<!-- Oracle Remote Diagnostic Agent / Data Collection Results 4.23 -->
<!-- Module:S400RAC Version:2.25 Report:crs_log OS:linux -->
---+!! CRS Log Files
%TOC%
---+ Last 2000 Lines of crsd.log File
---## Information Taken from /u01/crs/11.1.0/db_1/log/rac02/crsd/crsd.log
<verbatim>
2011-04-19 12:51:02.469: [ COMMCRS][1433344320]clsc_connect: (0x26d6920) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:02.469: [  CRSEVT][1096120640] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:02.640: [  CRSRES][1435445568] Stop of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-19 12:51:02.645: [  CRSRES][1408166208] Stop of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-19 12:51:02.807: [ COMMCRS][1433344320]clsc_connect: (0x7f23c80066e0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:02.807: [  CRSEVT][1435445568] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:02.812: [ COMMCRS][1370392896]clsc_connect: (0x26e55a0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:02.813: [  CRSEVT][1408166208] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:03.229: [ COMMCRS][1370392896]clsc_connect: (0x2638220) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:03.317: [ CRSMAIN][1098221888] UI Cmd received after shutdown began.
2011-04-19 12:51:03.348: [  CRSRES][1096120640] Stop of `pcard.listener` on member `rac02` succeeded.
2011-04-19 12:51:03.516: [ COMMCRS][1370392896]clsc_connect: (0x26e42a0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:03.516: [  CRSEVT][1096120640] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:03.537: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:03.572: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:03.598: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:03.623: [  CRSRES][1096120640] Attempting to stop `pcard.vip` on member `rac02`
2011-04-19 12:51:03.790: [ COMMCRS][1370392896]clsc_connect: (0x2638220) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:03.791: [  CRSEVT][1096120640] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:03.879: [  CRSRES][1096120640] Stop of `pcard.vip` on member `rac02` succeeded.
2011-04-19 12:51:04.047: [ COMMCRS][1370392896]clsc_connect: (0x26ab330) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:04.047: [  CRSEVT][1096120640] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:04.065: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:04.077: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:04.110: [  CRSRES][1096120640] Attempting to stop `pcard.db_pcard` on member `rac02`
2011-04-19 12:51:04.277: [ COMMCRS][1370392896]clsc_connect: (0x2576280) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:04.277: [  CRSEVT][1096120640] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:04.647: [ COMMCRS][1370392896]clsc_connect: (0x2638220) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:05.065: [ COMMCRS][1370392896]clsc_connect: (0x26abac0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:05.983: [ COMMCRS][1370392896]clsc_connect: (0x26d6920) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:06.651: [ COMMCRS][1370392896]clsc_connect: (0x26ac000) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:07.569: [ COMMCRS][1370392896]clsc_connect: (0x26e4590) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:08.487: [ COMMCRS][1370392896]clsc_connect: (0x2638220) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:09.905: [ COMMCRS][1370392896]clsc_connect: (0x26abac0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:10.572: [ COMMCRS][1370392896]clsc_connect: (0x26d6920) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:11.740: [ COMMCRS][1370392896]clsc_connect: (0x26d6920) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:13.157: [ COMMCRS][1370392896]clsc_connect: (0x26a4150) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:14.575: [ COMMCRS][1370392896]clsc_connect: (0x2575fb0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:15.993: [ COMMCRS][1370392896]clsc_connect: (0x26e4590) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:16.910: [ COMMCRS][1370392896]clsc_connect: (0x26ab330) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:17.328: [ COMMCRS][1370392896]clsc_connect: (0x26d6920) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:18.745: [ COMMCRS][1370392896]clsc_connect: (0x26e55a0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:19.913: [ COMMCRS][1370392896]clsc_connect: (0x2638220) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:20.331: [ COMMCRS][1370392896]clsc_connect: (0x26a3fd0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:21.748: [ COMMCRS][1370392896]clsc_connect: (0x26ac000) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:22.666: [ COMMCRS][1370392896]clsc_connect: (0x26abac0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:23.834: [ COMMCRS][1370392896]clsc_connect: (0x26d6aa0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:24.751: [ COMMCRS][1370392896]clsc_connect: (0x26a4150) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:25.919: [ COMMCRS][1370392896]clsc_connect: (0x2575fb0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:27.087: [ COMMCRS][1370392896]clsc_connect: (0x2575fb0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:28.004: [ COMMCRS][1370392896]clsc_connect: (0x26ac000) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:28.672: [ COMMCRS][1370392896]clsc_connect: (0x26d6aa0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:29.089: [ COMMCRS][1370392896]clsc_connect: (0x26ab330) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:30.007: [ COMMCRS][1370392896]clsc_connect: (0x2638220) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:30.425: [ COMMCRS][1370392896]clsc_connect: (0x2575fb0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:31.843: [ COMMCRS][1370392896]clsc_connect: (0x26a3fd0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:32.260: [ COMMCRS][1370392896]clsc_connect: (0x26aa230) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:32.806: [  CRSRES][1096120640] Stop of `pcard.db_pcard` on member `rac02` succeeded.
2011-04-19 12:51:32.929: [ COMMCRS][1370392896]clsc_connect: (0x26d6920) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:32.974: [ COMMCRS][1433344320]clsc_connect: (0x26e55a0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:32.975: [  CRSEVT][1096120640] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:33.005: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:33.015: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:33.048: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:33.063: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:33.119: [  CRSRES][1096120640] StopResource: setting CLI values
2011-04-19 12:51:33.138: [  CRSRES][1096120640] Attempting to stop `pcard` on member `rac02`
2011-04-19 12:51:33.305: [ COMMCRS][1433344320]clsc_connect: (0x2576280) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:33.305: [  CRSEVT][1096120640] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:33.328: [  CRSRES][1096120640] Stop of `pcard` on member `rac02` succeeded.
2011-04-19 12:51:33.496: [ COMMCRS][1433344320]clsc_connect: (0x26814b0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:33.496: [  CRSEVT][1096120640] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:33.512: [  CRSRES][1408166208] StopResource: setting CLI values
2011-04-19 12:51:33.529: [  CRSRES][1408166208] Attempting to stop `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-19 12:51:33.696: [ COMMCRS][1433344320]clsc_connect: (0x24cf360) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:33.696: [  CRSEVT][1408166208] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:34.347: [ COMMCRS][1433344320]clsc_connect: (0x26e42a0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:34.764: [ COMMCRS][1433344320]clsc_connect: (0x26252f0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:36.182: [ COMMCRS][1433344320]clsc_connect: (0x26ab1b0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:36.850: [ COMMCRS][1433344320]clsc_connect: (0x26a4150) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:37.319: [ CRSMAIN][1098221888] UI Cmd received after shutdown began.
2011-04-19 12:51:37.640: [  CRSRES][1408166208] Stop of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-19 12:51:37.807: [ COMMCRS][1433344320]clsc_connect: (0x2575290) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:37.807: [  CRSEVT][1408166208] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:37.825: [  CRSRES][1408166208] StopResource: setting CLI values
2011-04-19 12:51:37.852: [  CRSRES][1408166208] StopResource: setting CLI values
2011-04-19 12:51:37.870: [  CRSRES][1408166208] Attempting to stop `ora.rac02.vip` on member `rac02`
2011-04-19 12:51:38.037: [ COMMCRS][1433344320]clsc_connect: (0x24cf360) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:38.038: [  CRSEVT][1408166208] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:38.268: [ COMMCRS][1433344320]clsc_connect: (0x2576280) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:38.646: [  CRSRES][1408166208] Stop of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-19 12:51:38.813: [ COMMCRS][1433344320]clsc_connect: (0x2575290) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:51:38.813: [  CRSEVT][1408166208] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-19 12:51:39.105: [ CSSCLNT][1336822080]clssgsGGetStatus: CSS shutting down.

2011-04-19 12:51:39.105: [ CSSCLNT][1336822080]clssgsGroupGetStatus: returning 22

2011-04-19 12:51:39.105: [    CRSD][1336822080] CRSD exiting on CSS shutdown request
2011-04-19 12:51:39.105: [    CRSD][1336822080] Done.

2011-04-19 12:56:40.485: [ default][1900111600] CRS Daemon Starting
2011-04-19 12:56:40.498: [ CRSMAIN][1900111600] Checking the OCR device
2011-04-19 12:56:40.551: [ CRSMAIN][1900111600] Connecting to the CSS Daemon
2011-04-19 12:56:40.718: [ COMMCRS][1086163264]clsc_connect: (0xdfb3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-19 12:56:40.719: [ CSSCLNT][1900111600]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-19 12:56:40.719: [  CRSRTI][1900111600] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-19 12:56:41.886: [ COMMCRS][1086163264]clsc_connect: (0xdfb3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-19 12:56:41.886: [ CSSCLNT][1900111600]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-19 12:56:41.886: [  CRSRTI][1900111600] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-19 12:56:43.053: [ COMMCRS][1086163264]clsc_connect: (0xdfb3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-19 12:56:43.053: [ CSSCLNT][1900111600]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-19 12:56:43.054: [  CRSRTI][1900111600] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-19 12:56:44.220: [ COMMCRS][1086163264]clsc_connect: (0xdfb3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-19 12:56:44.220: [ CSSCLNT][1900111600]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-19 12:56:44.221: [  CRSRTI][1900111600] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-19 12:56:46.340: [ CRSMAIN][1900111600] CRSD running as the Privileged user

2011-04-19 12:56:46.578: [  CLSVER][1900111600] Active Version from OCR:11.1.0.7.0
2011-04-19 12:56:46.578: [  CLSVER][1900111600] Active Version and Software Version are same
2011-04-19 12:56:46.578: [ CRSMAIN][1900111600] Initializing OCR
2011-04-19 12:56:46.631: [  OCRRAW][1900111600]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-19 12:56:46.694: [    CRSD][1900111600] ENV Logging level for Module: allcomp  0
2011-04-19 12:56:46.696: [    CRSD][1900111600] ENV Logging level for Module: default  0
2011-04-19 12:56:46.698: [    CRSD][1900111600] ENV Logging level for Module: COMMCRS  0
2011-04-19 12:56:46.700: [    CRSD][1900111600] ENV Logging level for Module: COMMNS  0
2011-04-19 12:56:46.702: [    CRSD][1900111600] ENV Logging level for Module: CRSUI  0
2011-04-19 12:56:46.704: [    CRSD][1900111600] ENV Logging level for Module: CRSCOMM  0
2011-04-19 12:56:46.705: [    CRSD][1900111600] ENV Logging level for Module: CRSRTI  0
2011-04-19 12:56:46.707: [    CRSD][1900111600] ENV Logging level for Module: CRSMAIN  0
2011-04-19 12:56:46.708: [    CRSD][1900111600] ENV Logging level for Module: CRSPLACE  0
2011-04-19 12:56:46.710: [    CRSD][1900111600] ENV Logging level for Module: CRSAPP  0
2011-04-19 12:56:46.712: [    CRSD][1900111600] ENV Logging level for Module: CRSRES  0
2011-04-19 12:56:46.714: [    CRSD][1900111600] ENV Logging level for Module: CRSOCR  0
2011-04-19 12:56:46.716: [    CRSD][1900111600] ENV Logging level for Module: CRSTIMER  0
2011-04-19 12:56:46.718: [    CRSD][1900111600] ENV Logging level for Module: CRSEVT  0
2011-04-19 12:56:46.719: [    CRSD][1900111600] ENV Logging level for Module: CRSD  0
2011-04-19 12:56:46.721: [    CRSD][1900111600] ENV Logging level for Module: CLUCLS  0
2011-04-19 12:56:46.723: [    CRSD][1900111600] ENV Logging level for Module: CLSVER  0
2011-04-19 12:56:46.724: [    CRSD][1900111600] ENV Logging level for Module: OCRRAW  0
2011-04-19 12:56:46.726: [    CRSD][1900111600] ENV Logging level for Module: OCROSD  0
2011-04-19 12:56:46.728: [    CRSD][1900111600] ENV Logging level for Module: OCRCAC  0
2011-04-19 12:56:46.729: [    CRSD][1900111600] ENV Logging level for Module: CSSCLNT  0
2011-04-19 12:56:46.731: [    CRSD][1900111600] ENV Logging level for Module: OCRAPI  0
2011-04-19 12:56:46.733: [    CRSD][1900111600] ENV Logging level for Module: OCRUTL  0
2011-04-19 12:56:46.734: [    CRSD][1900111600] ENV Logging level for Module: OCRMSG  0
2011-04-19 12:56:46.736: [    CRSD][1900111600] ENV Logging level for Module: OCRCLI  0
2011-04-19 12:56:46.738: [    CRSD][1900111600] ENV Logging level for Module: OCRSRV  0
2011-04-19 12:56:46.739: [    CRSD][1900111600] ENV Logging level for Module: OCRMAS  0
2011-04-19 12:56:46.740: [ CRSMAIN][1900111600] Filename is /u01/crs/11.1.0/db_1/crs/init/rac02.pid
[  clsdmt][1337944384]Listening to (ADDRESS=(PROTOCOL=ipc)(KEY=rac02DBG_CRSD))
2011-04-19 12:56:46.770: [ CRSMAIN][1900111600] Using Authorizer location: /u01/crs/11.1.0/db_1/crs/auth/
2011-04-19 12:56:46.802: [ CRSMAIN][1900111600] Initializing RTI
2011-04-19 12:56:46.837: [ CRSMAIN][1900111600] Initializing EVMMgr
2011-04-19 12:56:46.837: [CRSTIMER][1354729792] Timer Thread Starting.
2011-04-19 12:56:47.004: [ COMMCRS][1363122496]clsc_connect: (0x7f736c028fe0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 12:56:48.332: [ CRSMAIN][1900111600] CRSD locked during state recovery, please wait.
2011-04-19 12:56:48.540: [ CRSMAIN][1900111600] CRSD recovered, unlocked.
2011-04-19 12:56:48.542: [ CRSMAIN][1900111600] QS socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=ora_crsqs))
2011-04-19 12:56:48.556: [ CRSMAIN][1900111600] CRSD UI socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=CRSD_UI_SOCKET))
2011-04-19 12:56:48.559: [ CRSMAIN][1900111600] E2E socket on: (ADDRESS=(PROTOCOL=tcp)(HOST=rac02-priv)(PORT=49896))
2011-04-19 12:56:48.559: [ CRSMAIN][1900111600] Starting Threads
2011-04-19 12:56:48.559: [ CRSMAIN][1400895808] Starting runCommandServer for (UI = 1, E2E = 0). 1
2011-04-19 12:56:48.559: [ CRSMAIN][1900111600] CRS Daemon Started.
2011-04-19 12:56:48.559: [ CRSMAIN][1398794560] Starting runCommandServer for (UI = 1, E2E = 0). 0
2011-04-19 12:56:48.645: [  CRSRES][1900111600]  startup = 1
2011-04-19 12:56:48.672: [  CRSRES][1900111600]  startup = 1
2011-04-19 12:56:48.716: [  CRSRES][1900111600]  startup = 1
2011-04-19 12:56:48.748: [  CRSRES][1900111600]  startup = 1
2011-04-19 12:56:48.770: [  CRSRES][1900111600]  startup = 1
2011-04-19 12:56:48.886: [  CRSRES][1407199552] StopResource: setting CLI values
2011-04-19 12:56:48.908: [  CRSRES][1407199552] Attempting to stop `ora.rac02.vip` on member `rac01`
2011-04-19 12:56:49.201: [  CRSRES][1407199552] Stop of `ora.rac02.vip` on member `rac01` succeeded.
2011-04-19 12:56:49.222: [  CRSRES][1407199552] startRunnable: setting CLI values
2011-04-19 12:56:49.223: [  CRSRES][1407199552] Attempting to start `ora.rac02.vip` on member `rac02`
2011-04-19 12:56:52.883: [  CRSRES][1407199552] Start of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-19 12:56:52.941: [  CRSRES][1407199552] startRunnable: setting CLI values
2011-04-19 12:56:52.950: [  CRSRES][1407199552] Attempting to start `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-19 12:56:54.032: [  CRSRES][1407199552] Start of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-19 12:56:54.091: [  CRSRES][1440782656] CRS-1002: Resource 'ora.rac02.LISTENER_RAC02.lsnr' is already running on member 'rac02'

2011-04-19 12:56:54.171: [  CRSRES][1407199552] StopResource: setting CLI values
2011-04-19 12:56:54.209: [  CRSRES][1436580160] startRunnable: setting CLI values
2011-04-19 12:56:54.226: [  CRSRES][1434478912] startRunnable: setting CLI values
2011-04-19 12:56:54.236: [  CRSRES][1436580160] Attempting to start `ora.rac02.ons` on member `rac02`
2011-04-19 12:56:54.238: [  CRSRES][1407199552] Attempting to stop `pcard.listener` on member `rac01`
2011-04-19 12:56:54.239: [  CRSRES][1434478912] Attempting to start `ora.rac02.gsd` on member `rac02`
2011-04-19 12:56:54.287: [  CRSRES][1407199552] Stop of `pcard.listener` on member `rac01` succeeded.
2011-04-19 12:56:54.288: [  CRSRES][1407199552] StopResource: setting CLI values
2011-04-19 12:56:54.314: [  CRSRES][1407199552] Attempting to stop `pcard.vip` on member `rac01`
2011-04-19 12:56:54.406: [  CRSRES][1407199552] Stop of `pcard.vip` on member `rac01` succeeded.
2011-04-19 12:56:54.406: [  CRSRES][1407199552] StopResource: setting CLI values
2011-04-19 12:56:54.431: [  CRSRES][1407199552] Attempting to stop `pcard` on member `rac01`
2011-04-19 12:56:54.459: [  CRSRES][1407199552] Stop of `pcard` on member `rac01` succeeded.
2011-04-19 12:56:54.476: [  CRSRES][1407199552] startRunnable: setting CLI values
2011-04-19 12:56:54.477: [  CRSRES][1407199552] Attempting to start `pcard` on member `rac02`
2011-04-19 12:56:54.538: [  CRSRES][1407199552] Start of `pcard` on member `rac02` succeeded.
2011-04-19 12:56:54.553: [  CRSRES][1407199552] startRunnable: setting CLI values
2011-04-19 12:56:54.554: [  CRSRES][1407199552] Attempting to start `pcard.vip` on member `rac02`
2011-04-19 12:56:54.632: [  CRSRES][1434478912] Start of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-19 12:56:55.738: [  CRSRES][1436580160] Start of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-19 12:56:57.674: [  CRSRES][1407199552] Start of `pcard.vip` on member `rac02` succeeded.
2011-04-19 12:56:57.723: [  CRSRES][1407199552] startRunnable: setting CLI values
2011-04-19 12:56:57.724: [  CRSRES][1407199552] Attempting to start `pcard.listener` on member `rac02`
2011-04-19 12:56:57.844: [  CRSRES][1407199552] Start of `pcard.listener` on member `rac02` succeeded.
2011-04-19 12:56:57.867: [  CRSRES][1407199552] startRunnable: setting CLI values
2011-04-19 12:56:57.885: [  CRSRES][1407199552] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-19 12:57:22.990: [  CRSRES][1407199552] Start of `pcard.db_pcard` on member `rac02` succeeded.
2011-04-19 12:57:23.108: [  CRSRES][1407199552] startRunnable: setting CLI values
2011-04-19 12:57:23.114: [  CRSRES][1407199552] Attempting to start `pcard.head` on member `rac02`
2011-04-19 12:57:23.180: [  CRSRES][1407199552] Start of `pcard.head` on member `rac02` succeeded.
2011-04-19 12:59:20.707: [  OCRMAS][1186875712]th_master:13: I AM THE NEW OCR MASTER at incar 40. Node Number 2
2011-04-19 12:59:20.780: [  OCRRAW][1186875712]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-19 12:59:20.814: [  OCRMAS][1186875712]th_master: Deleted ver keys from cache (master)
2011-04-19 12:59:21.996: [ CRSCOMM][1438681408] CLEANUP: Searching for connections to failed node rac01
2011-04-19 12:59:21.996: [  CRSEVT][1438681408] Processing member leave for rac01, incarnation: 196435612
2011-04-19 12:59:21.997: [    CRSD][1438681408] SM: recovery in process: 8
2011-04-19 12:59:21.997: [  CRSEVT][1438681408] Do failover for: rac01
2011-04-19 12:59:22.049: [  CRSRES][1438681408]  startup = 0
2011-04-19 12:59:22.076: [  CRSRES][1438681408]  startup = 0
2011-04-19 12:59:22.084: [  CRSRES][1438681408]  startup = 0
2011-04-19 12:59:22.094: [  CRSRES][1438681408]  startup = 0
2011-04-19 12:59:22.187: [  CRSRES][1434478912] startRunnable: setting CLI values
2011-04-19 12:59:22.199: [  CRSRES][1434478912] Attempting to start `ora.rac01.vip` on member `rac02`
2011-04-19 12:59:25.550: [  CRSRES][1434478912] Start of `ora.rac01.vip` on member `rac02` succeeded.
2011-04-19 12:59:25.574: [  CRSEVT][1438681408] Post recovery done evmd event for: rac01
2011-04-19 12:59:25.574: [    CRSD][1438681408] SM: recoveryDone: 0
2011-04-19 12:59:25.586: [  CRSEVT][1438681408] Processing RecoveryDone
2011-04-19 12:59:59.412: [    CRSD][1398794560] SM: rE2Ec: 4
2011-04-19 12:59:59.413: [  CRSRES][1438681408] StopResource: setting CLI values
2011-04-19 12:59:59.701: [    CRSD][1438681408] SM:dE2Ec: all E2E cmds done. 0
2011-04-19 13:07:24.727: [  CRSAPP][1438681408] CheckResource error for pcard.db_pcard error code = 1
2011-04-19 13:07:24.730: [  CRSRES][1438681408] In stateChanged, pcard.db_pcard target is ONLINE
2011-04-19 13:07:24.730: [  CRSRES][1438681408] pcard.db_pcard on rac02 went OFFLINE unexpectedly
2011-04-19 13:07:24.730: [  CRSRES][1438681408] StopResource: setting CLI values
2011-04-19 13:07:24.742: [  CRSRES][1438681408] Attempting to stop `pcard.db_pcard` on member `rac02`
2011-04-19 13:07:24.872: [  CRSRES][1438681408] Stop of `pcard.db_pcard` on member `rac02` succeeded.
2011-04-19 13:07:24.873: [  CRSRES][1438681408] pcard.db_pcard RESTART_COUNT=0 RESTART_ATTEMPTS=5
2011-04-19 13:07:24.873: [  CRSRES][1438681408] Restarting pcard.db_pcard on rac02
2011-04-19 13:07:24.884: [  CRSRES][1438681408] startRunnable: setting CLI values
2011-04-19 13:07:24.885: [  CRSRES][1438681408] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-19 13:07:41.841: [  CRSRES][1438681408] Start of `pcard.db_pcard` on member `rac02` succeeded.
2011-04-19 13:07:41.842: [  CRSRES][1438681408] Successfully restarted pcard.db_pcard on rac02, RESTART_COUNT=1
2011-04-19 13:07:41.882: [  CRSRES][1438681408] pcard.db_pcard Updated LAST_RESTART time in ocr
2011-04-19 13:13:37.915: [ CRSCOMM][1438681408] CLEANUP: Searching for connections to failed node rac01
2011-04-19 13:13:37.915: [  CRSEVT][1438681408] Processing member leave for rac01, incarnation: 196435614
2011-04-19 13:13:37.916: [    CRSD][1438681408] SM: recovery in process: 8
2011-04-19 13:13:37.916: [  CRSEVT][1438681408] Do failover for: rac01
2011-04-19 13:13:37.953: [  CRSRES][1438681408]  startup = 0
2011-04-19 13:13:37.976: [  CRSRES][1438681408]  startup = 0
2011-04-19 13:13:37.984: [  CRSRES][1438681408]  startup = 0
2011-04-19 13:13:38.002: [  CRSRES][1438681408]  startup = 0
2011-04-19 13:13:38.055: [  CRSRES][1407199552] startRunnable: setting CLI values
2011-04-19 13:13:38.056: [  CRSRES][1407199552] Attempting to start `ora.rac01.vip` on member `rac02`
2011-04-19 13:13:41.319: [  CRSRES][1407199552] Start of `ora.rac01.vip` on member `rac02` succeeded.
2011-04-19 13:13:41.340: [  CRSEVT][1438681408] Post recovery done evmd event for: rac01
2011-04-19 13:13:41.340: [    CRSD][1438681408] SM: recoveryDone: 0
2011-04-19 13:13:41.341: [  CRSEVT][1438681408] Processing RecoveryDone
2011-04-19 13:18:09.029: [    CRSD][1398794560] SM: rE2Ec: 4
2011-04-19 13:18:09.030: [  CRSRES][1438681408] StopResource: setting CLI values
2011-04-19 13:18:09.298: [    CRSD][1438681408] SM:dE2Ec: all E2E cmds done. 0
2011-04-19 13:21:28.705: [    CRSD][1438681408] Shutdown Manager: Wait for recovery is complete.0
2011-04-19 13:21:28.705: [    CRSD][1438681408] Shutdown Manager: Wait for startup is complete
2011-04-19 13:21:28.705: [    CRSD][1438681408] Shutdown Manager: Wait for ongoing commands is complete
2011-04-19 13:21:28.717: [    CRSD][1438681408] SM: asked E2E to exit.
2011-04-19 13:21:28.717: [ CRSCOMM][1398794560] CLEANUP: quiesceing connection 0x7f736c09d8e0
2011-04-19 13:21:28.717: [ CRSMAIN][1398794560] runCommandServer for E2E exiting.
2011-04-19 13:21:28.717: [    CRSD][1438681408] SM: E2E port closed.3
2011-04-19 13:21:28.717: [    CRSD][1438681408] SM: stoppedResources_all_1 m_state = 3
2011-04-19 13:21:28.726: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:28.732: [  CRSRES][1436580160] StopResource: setting CLI values
2011-04-19 13:21:28.737: [  CRSRES][1398794560] StopResource: setting CLI values
2011-04-19 13:21:28.742: [  CRSRES][1434478912] Attempting to stop `pcard.head` on member `rac02`
2011-04-19 13:21:28.761: [  CRSRES][1436580160] Attempting to stop `ora.rac02.ons` on member `rac02`
2011-04-19 13:21:28.777: [  CRSRES][1398794560] Attempting to stop `ora.rac02.gsd` on member `rac02`
2011-04-19 13:21:28.792: [  CRSRES][1434478912] Stop of `pcard.head` on member `rac02` succeeded.
2011-04-19 13:21:28.804: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:28.828: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:28.845: [  CRSRES][1434478912] Attempting to stop `pcard.listener` on member `rac02`
2011-04-19 13:21:28.873: [ CRSMAIN][1400895808] UI Cmd received after shutdown began.
2011-04-19 13:21:28.918: [  CRSRES][1434478912] Stop of `pcard.listener` on member `rac02` succeeded.
2011-04-19 13:21:28.932: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:28.942: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:28.973: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:28.984: [  CRSRES][1434478912] Attempting to stop `pcard.vip` on member `rac02`
2011-04-19 13:21:29.107: [  CRSRES][1434478912] Stop of `pcard.vip` on member `rac02` succeeded.
2011-04-19 13:21:29.118: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:29.140: [  CRSRES][1398794560] Stop of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-19 13:21:29.165: [  CRSRES][1436580160] Stop of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-19 13:21:29.175: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:29.196: [  CRSRES][1434478912] Attempting to stop `pcard.db_pcard` on member `rac02`
2011-04-19 13:21:56.676: [  CRSRES][1434478912] Stop of `pcard.db_pcard` on member `rac02` succeeded.
2011-04-19 13:21:56.692: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:56.697: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:56.702: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:56.710: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:56.725: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:56.731: [  CRSRES][1434478912] Attempting to stop `pcard` on member `rac02`
2011-04-19 13:21:56.754: [  CRSRES][1434478912] Stop of `pcard` on member `rac02` succeeded.
2011-04-19 13:21:56.764: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:21:56.781: [  CRSRES][1434478912] Attempting to stop `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-19 13:22:03.933: [ CRSMAIN][1400895808] UI Cmd received after shutdown began.
2011-04-19 13:22:04.060: [  CRSRES][1434478912] Stop of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-19 13:22:04.069: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:22:04.098: [  CRSRES][1434478912] StopResource: setting CLI values
2011-04-19 13:22:04.109: [  CRSRES][1434478912] Attempting to stop `ora.rac02.vip` on member `rac02`
2011-04-19 13:22:04.370: [  CRSRES][1434478912] Stop of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-19 13:22:04.494: [  CRSEVT][1402997056] Error dispatching EVM event; reconnecting
2011-04-19 13:22:04.525: [ CSSCLNT][1186875712]clssgsGGetStatus: CSS shutting down.

2011-04-19 13:22:04.525: [ CSSCLNT][1186875712]clssgsGroupGetStatus: returning 22

2011-04-19 13:22:04.525: [  OCRMAS][1186875712]th_master:21:clssgsgrpstat failed css ret code = 22
[  OCRAPI][1186875712]procr_ctx_set_invalid_no_abort: ctx set to invalid
2011-04-19 13:22:04.564: [ CSSCLNT][1329551680]clssgsGGetStatus: CSS shutting down.

2011-04-19 13:22:04.564: [ CSSCLNT][1329551680]clssgsGroupGetStatus: returning 22

2011-04-19 13:22:04.564: [    CRSD][1329551680] CRSD exiting on CSS shutdown request
2011-04-19 13:22:04.564: [    CRSD][1329551680] Done.

2011-04-19 13:23:06.262: [ default][3001329392] CRS Daemon Starting
2011-04-19 13:23:06.262: [ CRSMAIN][3001329392] Checking the OCR device
2011-04-19 13:23:06.282: [ CRSMAIN][3001329392] Connecting to the CSS Daemon
2011-04-19 13:23:06.450: [ COMMCRS][1106004288]clsc_connect: (0x26af3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-19 13:23:06.451: [ CSSCLNT][3001329392]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-19 13:23:06.451: [  CRSRTI][3001329392] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-19 13:23:07.617: [ COMMCRS][1106004288]clsc_connect: (0x26af3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-19 13:23:07.617: [ CSSCLNT][3001329392]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-19 13:23:07.617: [  CRSRTI][3001329392] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-19 13:23:08.785: [ COMMCRS][1106004288]clsc_connect: (0x26af3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-19 13:23:08.785: [ CSSCLNT][3001329392]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-19 13:23:08.786: [  CRSRTI][3001329392] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-19 13:23:10.726: [ CRSMAIN][3001329392] CRSD running as the Privileged user

2011-04-19 13:23:10.958: [  CLSVER][3001329392] Active Version from OCR:11.1.0.7.0
2011-04-19 13:23:10.958: [  CLSVER][3001329392] Active Version and Software Version are same
2011-04-19 13:23:10.958: [ CRSMAIN][3001329392] Initializing OCR
2011-04-19 13:23:11.059: [  OCRRAW][3001329392]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-19 13:23:11.183: [    CRSD][3001329392] ENV Logging level for Module: allcomp  0
2011-04-19 13:23:11.186: [    CRSD][3001329392] ENV Logging level for Module: default  0
2011-04-19 13:23:11.189: [    CRSD][3001329392] ENV Logging level for Module: COMMCRS  0
2011-04-19 13:23:11.192: [    CRSD][3001329392] ENV Logging level for Module: COMMNS  0
2011-04-19 13:23:11.195: [    CRSD][3001329392] ENV Logging level for Module: CRSUI  0
2011-04-19 13:23:11.197: [    CRSD][3001329392] ENV Logging level for Module: CRSCOMM  0
2011-04-19 13:23:11.199: [    CRSD][3001329392] ENV Logging level for Module: CRSRTI  0
2011-04-19 13:23:11.220: [    CRSD][3001329392] ENV Logging level for Module: CRSMAIN  0
2011-04-19 13:23:11.223: [    CRSD][3001329392] ENV Logging level for Module: CRSPLACE  0
2011-04-19 13:23:11.246: [    CRSD][3001329392] ENV Logging level for Module: CRSAPP  0
2011-04-19 13:23:11.249: [    CRSD][3001329392] ENV Logging level for Module: CRSRES  0
2011-04-19 13:23:11.251: [    CRSD][3001329392] ENV Logging level for Module: CRSOCR  0
2011-04-19 13:23:11.254: [    CRSD][3001329392] ENV Logging level for Module: CRSTIMER  0
2011-04-19 13:23:11.257: [    CRSD][3001329392] ENV Logging level for Module: CRSEVT  0
2011-04-19 13:23:11.258: [    CRSD][3001329392] ENV Logging level for Module: CRSD  0
2011-04-19 13:23:11.260: [    CRSD][3001329392] ENV Logging level for Module: CLUCLS  0
2011-04-19 13:23:11.263: [    CRSD][3001329392] ENV Logging level for Module: CLSVER  0
2011-04-19 13:23:11.267: [    CRSD][3001329392] ENV Logging level for Module: OCRRAW  0
2011-04-19 13:23:11.279: [    CRSD][3001329392] ENV Logging level for Module: OCROSD  0
2011-04-19 13:23:11.283: [    CRSD][3001329392] ENV Logging level for Module: OCRCAC  0
2011-04-19 13:23:11.299: [    CRSD][3001329392] ENV Logging level for Module: CSSCLNT  0
2011-04-19 13:23:11.305: [    CRSD][3001329392] ENV Logging level for Module: OCRAPI  0
2011-04-19 13:23:11.313: [    CRSD][3001329392] ENV Logging level for Module: OCRUTL  0
2011-04-19 13:23:11.329: [    CRSD][3001329392] ENV Logging level for Module: OCRMSG  0
2011-04-19 13:23:11.331: [    CRSD][3001329392] ENV Logging level for Module: OCRCLI  0
2011-04-19 13:23:11.334: [    CRSD][3001329392] ENV Logging level for Module: OCRSRV  0
2011-04-19 13:23:11.336: [    CRSD][3001329392] ENV Logging level for Module: OCRMAS  0
2011-04-19 13:23:11.336: [ CRSMAIN][3001329392] Filename is /u01/crs/11.1.0/db_1/crs/init/rac02.pid
[  clsdmt][1341000000]Listening to (ADDRESS=(PROTOCOL=ipc)(KEY=rac02DBG_CRSD))
2011-04-19 13:23:11.358: [ CRSMAIN][3001329392] Using Authorizer location: /u01/crs/11.1.0/db_1/crs/auth/
2011-04-19 13:23:11.404: [ CRSMAIN][3001329392] Initializing RTI
2011-04-19 13:23:11.453: [ CRSMAIN][3001329392] Initializing EVMMgr
2011-04-19 13:23:11.453: [CRSTIMER][1357785408] Timer Thread Starting.
2011-04-19 13:23:11.620: [ COMMCRS][1366178112]clsc_connect: (0x7f19ac0267f0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-19 13:23:13.045: [ CRSMAIN][3001329392] CRSD locked during state recovery, please wait.
2011-04-19 13:23:13.493: [ CRSMAIN][3001329392] CRSD recovered, unlocked.
2011-04-19 13:23:13.494: [ CRSMAIN][3001329392] QS socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=ora_crsqs))
2011-04-19 13:23:13.511: [ CRSMAIN][3001329392] CRSD UI socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=CRSD_UI_SOCKET))
2011-04-19 13:23:13.520: [ CRSMAIN][3001329392] E2E socket on: (ADDRESS=(PROTOCOL=tcp)(HOST=rac02-priv)(PORT=49896))
2011-04-19 13:23:13.520: [ CRSMAIN][3001329392] Starting Threads
2011-04-19 13:23:13.520: [ CRSMAIN][1095350592] Starting runCommandServer for (UI = 1, E2E = 0). 0
2011-04-19 13:23:13.520: [ CRSMAIN][3001329392] CRS Daemon Started.
2011-04-19 13:23:13.520: [ CRSMAIN][1401850176] Starting runCommandServer for (UI = 1, E2E = 0). 1
2011-04-19 13:23:13.590: [  CRSRES][3001329392]  startup = 1
2011-04-19 13:23:13.640: [  CRSRES][3001329392]  startup = 1
2011-04-19 13:23:13.719: [  CRSRES][3001329392]  startup = 1
2011-04-19 13:23:13.809: [  CRSRES][1406052672] StopResource: setting CLI values
2011-04-19 13:23:13.841: [  CRSRES][1406052672] Attempting to stop `ora.rac02.vip` on member `rac01`
2011-04-19 13:23:14.120: [  CRSRES][1406052672] Stop of `ora.rac02.vip` on member `rac01` succeeded.
2011-04-19 13:23:14.140: [  CRSRES][1406052672] startRunnable: setting CLI values
2011-04-19 13:23:14.140: [  CRSRES][1406052672] Attempting to start `ora.rac02.vip` on member `rac02`
2011-04-19 13:23:17.490: [  CRSRES][1406052672] Start of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-19 13:23:17.597: [  CRSRES][1406052672] startRunnable: setting CLI values
2011-04-19 13:23:17.629: [  CRSRES][1406052672] Attempting to start `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-19 13:23:18.079: [  CRSRES][1406052672] Start of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-19 13:23:18.148: [  CRSRES][1406052672] startRunnable: setting CLI values
2011-04-19 13:23:18.163: [  CRSRES][1406052672] Attempting to start `ora.rac02.gsd` on member `rac02`
2011-04-19 13:23:18.263: [  CRSRES][1433332032] startRunnable: setting CLI values
2011-04-19 13:23:18.287: [  CRSRES][1433332032] Attempting to start `ora.rac02.ons` on member `rac02`
2011-04-19 13:23:18.732: [  CRSRES][1406052672] Start of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-19 13:23:18.771: [  CRSRES][1435433280] CRS-1002: Resource 'ora.rac02.LISTENER_RAC02.lsnr' is already running on member 'rac02'

2011-04-19 13:23:19.802: [  CRSRES][1433332032] Start of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-19 14:12:45.683: [  OCRMAS][1189931328]th_master:13: I AM THE NEW OCR MASTER at incar 46. Node Number 2
2011-04-19 14:12:45.702: [  OCRRAW][1189931328]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-19 14:12:45.719: [  OCRMAS][1189931328]th_master: Deleted ver keys from cache (master)
2011-04-19 14:12:47.438: [ CRSCOMM][1433332032] CLEANUP: Searching for connections to failed node rac01
2011-04-19 14:12:47.438: [  CRSEVT][1433332032] Processing member leave for rac01, incarnation: 196435618
2011-04-19 14:12:47.439: [    CRSD][1433332032] SM: recovery in process: 8
2011-04-19 14:12:47.439: [  CRSEVT][1433332032] Do failover for: rac01
2011-04-19 14:12:47.522: [  CRSRES][1433332032]  startup = 0
2011-04-19 14:12:47.574: [  CRSRES][1433332032]  startup = 0
2011-04-19 14:12:47.681: [  CRSRES][1433332032]  startup = 0
2011-04-19 14:12:47.713: [  CRSRES][1433332032]  startup = 0
2011-04-19 14:12:47.765: [  CRSRES][1433332032]  startup = 0
2011-04-19 14:12:47.789: [  CRSRES][1433332032]  startup = 0
2011-04-19 14:12:47.820: [  CRSRES][1433332032]  startup = 0
2011-04-19 14:12:47.883: [  CRSRES][1433332032]  startup = 0
2011-04-19 14:12:47.926: [  CRSRES][1433332032]  startup = 0
2011-04-19 14:12:48.313: [  CRSRES][1435433280] startRunnable: setting CLI values
2011-04-19 14:12:48.336: [  CRSRES][1406052672] startRunnable: setting CLI values
2011-04-19 14:12:48.356: [  CRSRES][1435433280] Attempting to start `pcard` on member `rac02`
2011-04-19 14:12:48.360: [  CRSRES][1406052672] Attempting to start `ora.rac01.vip` on member `rac02`
2011-04-19 14:12:48.599: [  CRSRES][1435433280] Start of `pcard` on member `rac02` succeeded.
2011-04-19 14:12:48.661: [  CRSRES][1435433280] startRunnable: setting CLI values
2011-04-19 14:12:48.719: [  CRSRES][1435433280] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-19 14:12:51.984: [  CRSRES][1406052672] Start of `ora.rac01.vip` on member `rac02` succeeded.
2011-04-19 14:13:04.636: [  CRSRES][1435433280] Start of `pcard.db_pcard` on member `rac02` succeeded.
2011-04-19 14:13:04.707: [  CRSRES][1435433280] startRunnable: setting CLI values
2011-04-19 14:13:04.727: [  CRSRES][1435433280] Attempting to start `pcard.vip` on member `rac02`
2011-04-19 14:13:07.897: [  CRSRES][1435433280] Start of `pcard.vip` on member `rac02` succeeded.
2011-04-19 14:13:07.933: [  CRSRES][1435433280] startRunnable: setting CLI values
2011-04-19 14:13:07.943: [  CRSRES][1435433280] Attempting to start `pcard.listener` on member `rac02`
2011-04-19 14:13:08.095: [  CRSRES][1435433280] Start of `pcard.listener` on member `rac02` succeeded.
2011-04-19 14:13:08.222: [  CRSRES][1435433280] startRunnable: setting CLI values
2011-04-19 14:13:08.247: [  CRSRES][1435433280] Attempting to start `pcard.head` on member `rac02`
2011-04-19 14:13:08.330: [  CRSRES][1435433280] Start of `pcard.head` on member `rac02` succeeded.
2011-04-19 14:13:08.331: [  CRSEVT][1433332032] Post recovery done evmd event for: rac01
2011-04-19 14:13:08.332: [    CRSD][1433332032] SM: recoveryDone: 0
2011-04-19 14:13:08.333: [  CRSEVT][1433332032] Processing RecoveryDone
2011-04-19 14:18:00.125: [    CRSD][1095350592] SM: rE2Ec: 4
2011-04-19 14:18:00.126: [  CRSRES][1448028480] StopResource: setting CLI values
2011-04-19 14:18:00.402: [    CRSD][1448028480] SM:dE2Ec: all E2E cmds done. 0
2011-04-20 08:36:56.010: [  CRSRES][1445927232] StopResource: setting CLI values
2011-04-20 08:36:56.029: [  CRSRES][1445927232] Attempting to stop `pcard.head` on member `rac02`
2011-04-20 08:36:56.225: [  CRSRES][1445927232] Stop of `pcard.head` on member `rac02` succeeded.
2011-04-20 08:36:56.226: [  CRSRES][1445927232] StopResource: setting CLI values
2011-04-20 08:36:56.279: [  CRSRES][1445927232] Attempting to stop `pcard.db_pcard` on member `rac02`
2011-04-20 08:37:22.380: [  CRSRES][1445927232] Stop of `pcard.db_pcard` on member `rac02` succeeded.
2011-04-20 08:37:22.381: [  CRSRES][1445927232] StopResource: setting CLI values
2011-04-20 08:37:22.397: [  CRSRES][1445927232] Attempting to stop `pcard.listener` on member `rac02`
2011-04-20 08:37:22.439: [  CRSRES][1445927232] Stop of `pcard.listener` on member `rac02` succeeded.
2011-04-20 08:37:22.440: [  CRSRES][1445927232] StopResource: setting CLI values
2011-04-20 08:37:22.448: [  CRSRES][1445927232] Attempting to stop `pcard.vip` on member `rac02`
2011-04-20 08:37:22.522: [  CRSRES][1445927232] Stop of `pcard.vip` on member `rac02` succeeded.
2011-04-20 08:37:22.523: [  CRSRES][1445927232] StopResource: setting CLI values
2011-04-20 08:37:22.537: [  CRSRES][1445927232] Attempting to stop `pcard` on member `rac02`
2011-04-20 08:37:22.562: [  CRSRES][1445927232] Stop of `pcard` on member `rac02` succeeded.
2011-04-20 08:37:22.583: [  CRSRES][1445927232] Attempting to start `pcard` on member `rac01`
2011-04-20 08:37:22.650: [  CRSRES][1445927232] Start of `pcard` on member `rac01` succeeded.
2011-04-20 08:37:22.676: [  CRSRES][1445927232] Attempting to start `pcard.vip` on member `rac01`
2011-04-20 08:37:25.809: [  CRSRES][1445927232] Start of `pcard.vip` on member `rac01` succeeded.
2011-04-20 08:37:25.849: [  CRSRES][1445927232] Attempting to start `pcard.listener` on member `rac01`
2011-04-20 08:37:25.981: [  CRSRES][1445927232] Start of `pcard.listener` on member `rac01` succeeded.
2011-04-20 08:37:26.018: [  CRSRES][1445927232] Attempting to start `pcard.db_pcard` on member `rac01`
2011-04-20 08:37:46.617: [  CRSRES][1445927232] Start of `pcard.db_pcard` on member `rac01` succeeded.
2011-04-20 08:37:46.638: [  CRSRES][1445927232] Attempting to start `pcard.head` on member `rac01`
2011-04-20 08:37:46.712: [  CRSRES][1445927232] Start of `pcard.head` on member `rac01` succeeded.
2011-04-22 18:01:28.288: [  CRSEVT][1445927232] CAAMonitorHandler :: 0:Could not join /u01/crs/11.1.0/db_1/bin/racgwrap(check)
category: 1234, operation: scls_process_join, loc: childcrash, OS error: 0, other: Abnormal termination of the child

2011-04-22 18:01:28.288: [  CRSEVT][1445927232] CAAMonitorHandler :: 0:Action Script /u01/crs/11.1.0/db_1/bin/racgwrap(check) timed out for ora.rac02.vip! (timeout=60)
2011-04-22 18:01:28.288: [  CRSAPP][1445927232] CheckResource error for ora.rac02.vip error code = -2
2011-04-24 18:27:59.816: [  CRSEVT][1445927232] CAAMonitorHandler :: 0:Could not join /u01/crs/11.1.0/db_1/bin/racgwrap(check)
category: 1234, operation: scls_process_join, loc: childcrash, OS error: 0, other: Abnormal termination of the child

2011-04-24 18:27:59.816: [  CRSEVT][1445927232] CAAMonitorHandler :: 0:Action Script /u01/crs/11.1.0/db_1/bin/racgwrap(check) timed out for ora.rac02.vip! (timeout=60)
2011-04-24 18:27:59.817: [  CRSAPP][1445927232] CheckResource error for ora.rac02.vip error code = -2
2011-04-24 18:27:59.817: [  OCRSRV][1198324032]th_select_handler: Disconncting. conn [-1543232752] Req forsame client was already in queue.
2011-04-26 07:16:07.829: [ COMMCRS][1198324032]Authorization failed, network error

2011-04-26 07:16:07.832: [  OCRSRV][1198324032]th_select_answer: Failure in answer. clsc ret [3]
2011-04-26 07:16:37.832: [ COMMCRS][1198324032]Authorization failed, network error

2011-04-26 07:16:53.067: [  OCRSRV][1198324032]th_select_answer: Failure in answer. clsc ret [3]
2011-04-26 07:17:23.069: [ COMMCRS][1198324032]Authorization failed, network error

2011-04-26 07:17:23.069: [  OCRSRV][1198324032]th_select_answer: Failure in answer. clsc ret [3]
2011-04-26 07:18:38.319: [ COMMCRS][1198324032]Authorization failed, network error

2011-04-26 07:18:38.322: [  OCRSRV][1198324032]th_select_answer: Failure in answer. clsc ret [3]
2011-04-26 07:19:08.323: [ COMMCRS][1198324032]Authorization failed, network error

2011-04-26 07:19:14.024: [  OCRSRV][1198324032]th_select_answer: Failure in answer. clsc ret [3]
2011-04-26 07:19:53.563: [ COMMCRS][1198324032]Authorization failed, network error

2011-04-26 07:19:53.565: [  OCRSRV][1198324032]th_select_answer: Failure in answer. clsc ret [3]
2011-04-26 07:20:26.546: [  CRSEVT][1445927232] CAAMonitorHandler :: 0:Could not join /u01/crs/11.1.0/db_1/bin/racgwrap(check)
category: 1234, operation: scls_process_join, loc: childcrash, OS error: 0, other: Abnormal termination of the child

2011-04-26 07:20:26.546: [ COMMCRS][1282251072]clscsendx: (0x31c3c20) Connection not active

2011-04-26 07:20:26.546: [  CRSEVT][1445927232] CAAMonitorHandler :: 0:Action Script /u01/crs/11.1.0/db_1/bin/racgwrap(check) timed out for ora.rac02.vip! (timeout=60)
2011-04-26 07:20:26.546: [  CRSAPP][1445927232] CheckResource error for ora.rac02.vip error code = -2
2011-04-26 15:16:42.749: [ default][3034822384] CRS Daemon Starting
2011-04-26 15:16:42.758: [ CRSMAIN][3034822384] Checking the OCR device
2011-04-26 15:16:42.830: [ CRSMAIN][3034822384] Connecting to the CSS Daemon
2011-04-26 15:16:42.997: [ COMMCRS][1112435008]clsc_connect: (0x10c33d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-26 15:16:42.997: [ CSSCLNT][3034822384]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-26 15:16:42.998: [  CRSRTI][3034822384] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-26 15:16:44.168: [ COMMCRS][1112435008]clsc_connect: (0x10c33d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-26 15:16:44.168: [ CSSCLNT][3034822384]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-26 15:16:44.169: [  CRSRTI][3034822384] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-26 15:16:45.337: [ COMMCRS][1112435008]clsc_connect: (0x10c33d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-26 15:16:45.337: [ CSSCLNT][3034822384]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-26 15:16:45.337: [  CRSRTI][3034822384] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-26 15:16:47.615: [ CRSMAIN][3034822384] CRSD running as the Privileged user

2011-04-26 15:16:47.810: [  CLSVER][3034822384] Active Version from OCR:11.1.0.7.0
2011-04-26 15:16:47.810: [  CLSVER][3034822384] Active Version and Software Version are same
2011-04-26 15:16:47.810: [ CRSMAIN][3034822384] Initializing OCR
2011-04-26 15:16:47.833: [  OCRRAW][3034822384]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-26 15:16:47.870: [    CRSD][3034822384] ENV Logging level for Module: allcomp  0
2011-04-26 15:16:47.872: [    CRSD][3034822384] ENV Logging level for Module: default  0
2011-04-26 15:16:47.875: [    CRSD][3034822384] ENV Logging level for Module: COMMCRS  0
2011-04-26 15:16:47.877: [    CRSD][3034822384] ENV Logging level for Module: COMMNS  0
2011-04-26 15:16:47.879: [    CRSD][3034822384] ENV Logging level for Module: CRSUI  0
2011-04-26 15:16:47.881: [    CRSD][3034822384] ENV Logging level for Module: CRSCOMM  0
2011-04-26 15:16:47.883: [    CRSD][3034822384] ENV Logging level for Module: CRSRTI  0
2011-04-26 15:16:47.885: [    CRSD][3034822384] ENV Logging level for Module: CRSMAIN  0
2011-04-26 15:16:47.887: [    CRSD][3034822384] ENV Logging level for Module: CRSPLACE  0
2011-04-26 15:16:47.889: [    CRSD][3034822384] ENV Logging level for Module: CRSAPP  0
2011-04-26 15:16:47.891: [    CRSD][3034822384] ENV Logging level for Module: CRSRES  0
2011-04-26 15:16:47.893: [    CRSD][3034822384] ENV Logging level for Module: CRSOCR  0
2011-04-26 15:16:47.895: [    CRSD][3034822384] ENV Logging level for Module: CRSTIMER  0
2011-04-26 15:16:47.897: [    CRSD][3034822384] ENV Logging level for Module: CRSEVT  0
2011-04-26 15:16:47.899: [    CRSD][3034822384] ENV Logging level for Module: CRSD  0
2011-04-26 15:16:47.901: [    CRSD][3034822384] ENV Logging level for Module: CLUCLS  0
2011-04-26 15:16:47.903: [    CRSD][3034822384] ENV Logging level for Module: CLSVER  0
2011-04-26 15:16:47.905: [    CRSD][3034822384] ENV Logging level for Module: OCRRAW  0
2011-04-26 15:16:47.909: [    CRSD][3034822384] ENV Logging level for Module: OCROSD  0
2011-04-26 15:16:47.912: [    CRSD][3034822384] ENV Logging level for Module: OCRCAC  0
2011-04-26 15:16:47.914: [    CRSD][3034822384] ENV Logging level for Module: CSSCLNT  0
2011-04-26 15:16:47.916: [    CRSD][3034822384] ENV Logging level for Module: OCRAPI  0
2011-04-26 15:16:47.917: [    CRSD][3034822384] ENV Logging level for Module: OCRUTL  0
2011-04-26 15:16:47.920: [    CRSD][3034822384] ENV Logging level for Module: OCRMSG  0
2011-04-26 15:16:47.922: [    CRSD][3034822384] ENV Logging level for Module: OCRCLI  0
2011-04-26 15:16:47.924: [    CRSD][3034822384] ENV Logging level for Module: OCRSRV  0
2011-04-26 15:16:47.926: [    CRSD][3034822384] ENV Logging level for Module: OCRMAS  0
2011-04-26 15:16:47.927: [ CRSMAIN][3034822384] Filename is /u01/crs/11.1.0/db_1/crs/init/rac02.pid
[  clsdmt][1347430720]Listening to (ADDRESS=(PROTOCOL=ipc)(KEY=rac02DBG_CRSD))
2011-04-26 15:16:47.977: [ CRSMAIN][3034822384] Using Authorizer location: /u01/crs/11.1.0/db_1/crs/auth/
2011-04-26 15:16:47.998: [ CRSMAIN][3034822384] Initializing RTI
2011-04-26 15:16:48.046: [ CRSMAIN][3034822384] Initializing EVMMgr
2011-04-26 15:16:48.046: [CRSTIMER][1364216128] Timer Thread Starting.
2011-04-26 15:16:48.213: [ COMMCRS][1372608832]clsc_connect: (0x7f02b00175d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-26 15:16:49.776: [ CRSMAIN][3034822384] CRSD locked during state recovery, please wait.
2011-04-26 15:16:49.948: [ CRSMAIN][3034822384] CRSD recovered, unlocked.
2011-04-26 15:16:49.950: [ CRSMAIN][3034822384] QS socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=ora_crsqs))
2011-04-26 15:16:49.976: [ CRSMAIN][3034822384] CRSD UI socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=CRSD_UI_SOCKET))
2011-04-26 15:16:49.989: [ CRSMAIN][3034822384] E2E socket on: (ADDRESS=(PROTOCOL=tcp)(HOST=rac02-priv)(PORT=49896))
2011-04-26 15:16:49.989: [ CRSMAIN][3034822384] Starting Threads
2011-04-26 15:16:49.989: [ CRSMAIN][3034822384] CRS Daemon Started.
2011-04-26 15:16:49.989: [ CRSMAIN][1103141184] Starting runCommandServer for (UI = 1, E2E = 0). 0
2011-04-26 15:16:49.989: [ CRSMAIN][1408280896] Starting runCommandServer for (UI = 1, E2E = 0). 1
2011-04-26 15:16:50.067: [  CRSRES][3034822384]  startup = 1
2011-04-26 15:16:50.098: [  CRSRES][3034822384]  startup = 1
2011-04-26 15:16:50.156: [  CRSRES][3034822384]  startup = 1
2011-04-26 15:16:50.249: [  CRSRES][1100859712] StopResource: setting CLI values
2011-04-26 15:16:50.273: [  CRSRES][1100859712] Attempting to stop `ora.rac02.vip` on member `rac01`
2011-04-26 15:16:50.556: [  CRSRES][1100859712] Stop of `ora.rac02.vip` on member `rac01` succeeded.
2011-04-26 15:16:50.572: [  CRSRES][1100859712] startRunnable: setting CLI values
2011-04-26 15:16:50.572: [  CRSRES][1100859712] Attempting to start `ora.rac02.vip` on member `rac02`
2011-04-26 15:16:54.195: [  CRSRES][1100859712] Start of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-26 15:16:54.364: [  CRSRES][1100859712] startRunnable: setting CLI values
2011-04-26 15:16:54.398: [  CRSRES][1100859712] Attempting to start `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-26 15:16:55.612: [  CRSRES][1100859712] Start of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-26 15:16:55.724: [  CRSRES][1100859712] startRunnable: setting CLI values
2011-04-26 15:16:55.747: [  CRSRES][1100859712] Attempting to start `ora.rac02.ons` on member `rac02`
2011-04-26 15:16:55.809: [  CRSRES][1437661504] startRunnable: setting CLI values
2011-04-26 15:16:55.854: [  CRSRES][1437661504] Attempting to start `ora.rac02.gsd` on member `rac02`
2011-04-26 15:16:56.497: [  CRSRES][1437661504] Start of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-26 15:16:56.594: [  CRSRES][1441864000] CRS-1002: Resource 'ora.rac02.LISTENER_RAC02.lsnr' is already running on member 'rac02'

2011-04-26 15:16:57.609: [  CRSRES][1100859712] Start of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-27 05:12:30.328: [ default][3662677744] CRS Daemon Starting
2011-04-27 05:12:30.348: [ CRSMAIN][3662677744] Checking the OCR device
2011-04-27 05:12:30.382: [ CRSMAIN][3662677744] Connecting to the CSS Daemon
2011-04-27 05:12:30.557: [ COMMCRS][1088813376]clsc_connect: (0x1d9a3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 05:12:30.557: [ CSSCLNT][3662677744]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 05:12:30.557: [  CRSRTI][3662677744] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 05:12:31.724: [ COMMCRS][1088813376]clsc_connect: (0x1d9a3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 05:12:31.724: [ CSSCLNT][3662677744]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 05:12:31.724: [  CRSRTI][3662677744] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 05:12:32.892: [ COMMCRS][1088813376]clsc_connect: (0x1d9a3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 05:12:32.892: [ CSSCLNT][3662677744]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 05:12:32.892: [  CRSRTI][3662677744] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 05:12:34.060: [ COMMCRS][1088813376]clsc_connect: (0x1d9a3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 05:12:34.060: [ CSSCLNT][3662677744]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 05:12:34.060: [  CRSRTI][3662677744] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 05:12:35.805: [ CRSMAIN][3662677744] CRSD running as the Privileged user

2011-04-27 05:12:36.020: [  CLSVER][3662677744] Active Version from OCR:11.1.0.7.0
2011-04-27 05:12:36.020: [  CLSVER][3662677744] Active Version and Software Version are same
2011-04-27 05:12:36.020: [ CRSMAIN][3662677744] Initializing OCR
2011-04-27 05:12:36.056: [  OCRRAW][3662677744]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-27 05:12:36.113: [    CRSD][3662677744] ENV Logging level for Module: allcomp  0
2011-04-27 05:12:36.116: [    CRSD][3662677744] ENV Logging level for Module: default  0
2011-04-27 05:12:36.118: [    CRSD][3662677744] ENV Logging level for Module: COMMCRS  0
2011-04-27 05:12:36.120: [    CRSD][3662677744] ENV Logging level for Module: COMMNS  0
2011-04-27 05:12:36.123: [    CRSD][3662677744] ENV Logging level for Module: CRSUI  0
2011-04-27 05:12:36.126: [    CRSD][3662677744] ENV Logging level for Module: CRSCOMM  0
2011-04-27 05:12:36.129: [    CRSD][3662677744] ENV Logging level for Module: CRSRTI  0
2011-04-27 05:12:36.131: [    CRSD][3662677744] ENV Logging level for Module: CRSMAIN  0
2011-04-27 05:12:36.133: [    CRSD][3662677744] ENV Logging level for Module: CRSPLACE  0
2011-04-27 05:12:36.136: [    CRSD][3662677744] ENV Logging level for Module: CRSAPP  0
2011-04-27 05:12:36.139: [    CRSD][3662677744] ENV Logging level for Module: CRSRES  0
2011-04-27 05:12:36.142: [    CRSD][3662677744] ENV Logging level for Module: CRSOCR  0
2011-04-27 05:12:36.144: [    CRSD][3662677744] ENV Logging level for Module: CRSTIMER  0
2011-04-27 05:12:36.146: [    CRSD][3662677744] ENV Logging level for Module: CRSEVT  0
2011-04-27 05:12:36.148: [    CRSD][3662677744] ENV Logging level for Module: CRSD  0
2011-04-27 05:12:36.151: [    CRSD][3662677744] ENV Logging level for Module: CLUCLS  0
2011-04-27 05:12:36.153: [    CRSD][3662677744] ENV Logging level for Module: CLSVER  0
2011-04-27 05:12:36.155: [    CRSD][3662677744] ENV Logging level for Module: OCRRAW  0
2011-04-27 05:12:36.158: [    CRSD][3662677744] ENV Logging level for Module: OCROSD  0
2011-04-27 05:12:36.161: [    CRSD][3662677744] ENV Logging level for Module: OCRCAC  0
2011-04-27 05:12:36.163: [    CRSD][3662677744] ENV Logging level for Module: CSSCLNT  0
2011-04-27 05:12:36.165: [    CRSD][3662677744] ENV Logging level for Module: OCRAPI  0
2011-04-27 05:12:36.168: [    CRSD][3662677744] ENV Logging level for Module: OCRUTL  0
2011-04-27 05:12:36.170: [    CRSD][3662677744] ENV Logging level for Module: OCRMSG  0
2011-04-27 05:12:36.172: [    CRSD][3662677744] ENV Logging level for Module: OCRCLI  0
2011-04-27 05:12:36.175: [    CRSD][3662677744] ENV Logging level for Module: OCRSRV  0
2011-04-27 05:12:36.177: [    CRSD][3662677744] ENV Logging level for Module: OCRMAS  0
2011-04-27 05:12:36.177: [ CRSMAIN][3662677744] Filename is /u01/crs/11.1.0/db_1/crs/init/rac02.pid
[  clsdmt][1347365184]Listening to (ADDRESS=(PROTOCOL=ipc)(KEY=rac02DBG_CRSD))
2011-04-27 05:12:36.204: [ CRSMAIN][3662677744] Using Authorizer location: /u01/crs/11.1.0/db_1/crs/auth/
2011-04-27 05:12:36.254: [ CRSMAIN][3662677744] Initializing RTI
2011-04-27 05:12:36.327: [ CRSMAIN][3662677744] Initializing EVMMgr
2011-04-27 05:12:36.327: [CRSTIMER][1364150592] Timer Thread Starting.
2011-04-27 05:12:36.494: [ COMMCRS][1372543296]clsc_connect: (0x7f75d401ff10) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 05:12:37.162: [ COMMCRS][1372543296]clsc_connect: (0x7f75d4029ba0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 05:12:38.562: [ CRSMAIN][3662677744] CRSD locked during state recovery, please wait.
2011-04-27 05:12:38.787: [ CRSMAIN][3662677744] CRSD recovered, unlocked.
2011-04-27 05:12:38.788: [ CRSMAIN][3662677744] QS socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=ora_crsqs))
2011-04-27 05:12:38.799: [ CRSMAIN][3662677744] CRSD UI socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=CRSD_UI_SOCKET))
2011-04-27 05:12:38.801: [ CRSMAIN][3662677744] E2E socket on: (ADDRESS=(PROTOCOL=tcp)(HOST=rac02-priv)(PORT=49896))
2011-04-27 05:12:38.801: [ CRSMAIN][3662677744] Starting Threads
2011-04-27 05:12:38.802: [ CRSMAIN][1090914624] Starting runCommandServer for (UI = 1, E2E = 0). 0
2011-04-27 05:12:38.802: [ CRSMAIN][3662677744] CRS Daemon Started.
2011-04-27 05:12:38.802: [ CRSMAIN][1095342400] Starting runCommandServer for (UI = 1, E2E = 0). 1
2011-04-27 05:12:38.922: [  CRSRES][3662677744]  startup = 1
2011-04-27 05:12:38.948: [  CRSRES][3662677744]  startup = 1
2011-04-27 05:12:39.015: [  CRSRES][3662677744]  startup = 1
2011-04-27 05:12:39.120: [  CRSRES][1408215360] StopResource: setting CLI values
2011-04-27 05:12:39.148: [  CRSRES][1408215360] Attempting to stop `ora.rac02.vip` on member `rac01`
2011-04-27 05:12:39.442: [  CRSRES][1408215360] Stop of `ora.rac02.vip` on member `rac01` succeeded.
2011-04-27 05:12:39.466: [  CRSRES][1408215360] startRunnable: setting CLI values
2011-04-27 05:12:39.466: [  CRSRES][1408215360] Attempting to start `ora.rac02.vip` on member `rac02`
2011-04-27 05:12:42.927: [  CRSRES][1408215360] Start of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-27 05:12:42.984: [  CRSRES][1408215360] startRunnable: setting CLI values
2011-04-27 05:12:42.990: [  CRSRES][1408215360] Attempting to start `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-27 05:12:44.224: [  CRSRES][1408215360] Start of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-27 05:12:44.437: [  CRSRES][1408215360] startRunnable: setting CLI values
2011-04-27 05:12:44.459: [  CRSRES][1093015872] startRunnable: setting CLI values
2011-04-27 05:12:44.464: [  CRSRES][1408215360] Attempting to start `ora.rac02.gsd` on member `rac02`
2011-04-27 05:12:44.471: [  CRSRES][1093015872] Attempting to start `ora.rac02.ons` on member `rac02`
2011-04-27 05:12:44.840: [  CRSRES][1408215360] Start of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-27 05:12:45.177: [  CRSRES][1435494720] CRS-1002: Resource 'ora.rac02.LISTENER_RAC02.lsnr' is already running on member 'rac02'

2011-04-27 05:12:45.955: [  CRSRES][1093015872] Start of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-27 09:09:36.341: [ default][1829238512] CRS Daemon Starting
2011-04-27 09:09:36.363: [ CRSMAIN][1829238512] Checking the OCR device
2011-04-27 09:09:36.387: [ CRSMAIN][1829238512] Connecting to the CSS Daemon
2011-04-27 09:09:36.554: [ COMMCRS][1115572544]clsc_connect: (0x19993d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 09:09:36.554: [ CSSCLNT][1829238512]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 09:09:36.555: [  CRSRTI][1829238512] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 09:09:37.722: [ COMMCRS][1115572544]clsc_connect: (0x19993d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 09:09:37.722: [ CSSCLNT][1829238512]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 09:09:37.722: [  CRSRTI][1829238512] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 09:09:38.890: [ COMMCRS][1115572544]clsc_connect: (0x19993d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 09:09:38.890: [ CSSCLNT][1829238512]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 09:09:38.890: [  CRSRTI][1829238512] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 09:09:40.058: [ COMMCRS][1115572544]clsc_connect: (0x19993d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 09:09:40.058: [ CSSCLNT][1829238512]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 09:09:40.058: [  CRSRTI][1829238512] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 09:09:41.522: [ CRSMAIN][1829238512] CRSD running as the Privileged user

2011-04-27 09:09:41.775: [  CLSVER][1829238512] Active Version from OCR:11.1.0.7.0
2011-04-27 09:09:41.776: [  CLSVER][1829238512] Active Version and Software Version are same
2011-04-27 09:09:41.776: [ CRSMAIN][1829238512] Initializing OCR
2011-04-27 09:09:41.849: [  OCRRAW][1829238512]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-27 09:09:41.945: [    CRSD][1829238512] ENV Logging level for Module: allcomp  0
2011-04-27 09:09:41.947: [    CRSD][1829238512] ENV Logging level for Module: default  0
2011-04-27 09:09:41.949: [    CRSD][1829238512] ENV Logging level for Module: COMMCRS  0
2011-04-27 09:09:41.951: [    CRSD][1829238512] ENV Logging level for Module: COMMNS  0
2011-04-27 09:09:41.955: [    CRSD][1829238512] ENV Logging level for Module: CRSUI  0
2011-04-27 09:09:41.958: [    CRSD][1829238512] ENV Logging level for Module: CRSCOMM  0
2011-04-27 09:09:41.962: [    CRSD][1829238512] ENV Logging level for Module: CRSRTI  0
2011-04-27 09:09:41.965: [    CRSD][1829238512] ENV Logging level for Module: CRSMAIN  0
2011-04-27 09:09:41.967: [    CRSD][1829238512] ENV Logging level for Module: CRSPLACE  0
2011-04-27 09:09:41.969: [    CRSD][1829238512] ENV Logging level for Module: CRSAPP  0
2011-04-27 09:09:41.971: [    CRSD][1829238512] ENV Logging level for Module: CRSRES  0
2011-04-27 09:09:41.973: [    CRSD][1829238512] ENV Logging level for Module: CRSOCR  0
2011-04-27 09:09:41.976: [    CRSD][1829238512] ENV Logging level for Module: CRSTIMER  0
2011-04-27 09:09:41.978: [    CRSD][1829238512] ENV Logging level for Module: CRSEVT  0
2011-04-27 09:09:41.992: [    CRSD][1829238512] ENV Logging level for Module: CRSD  0
2011-04-27 09:09:41.994: [    CRSD][1829238512] ENV Logging level for Module: CLUCLS  0
2011-04-27 09:09:41.996: [    CRSD][1829238512] ENV Logging level for Module: CLSVER  0
2011-04-27 09:09:41.998: [    CRSD][1829238512] ENV Logging level for Module: OCRRAW  0
2011-04-27 09:09:42.000: [    CRSD][1829238512] ENV Logging level for Module: OCROSD  0
2011-04-27 09:09:42.003: [    CRSD][1829238512] ENV Logging level for Module: OCRCAC  0
2011-04-27 09:09:42.008: [    CRSD][1829238512] ENV Logging level for Module: CSSCLNT  0
2011-04-27 09:09:42.015: [    CRSD][1829238512] ENV Logging level for Module: OCRAPI  0
2011-04-27 09:09:42.017: [    CRSD][1829238512] ENV Logging level for Module: OCRUTL  0
2011-04-27 09:09:42.020: [    CRSD][1829238512] ENV Logging level for Module: OCRMSG  0
2011-04-27 09:09:42.023: [    CRSD][1829238512] ENV Logging level for Module: OCRCLI  0
2011-04-27 09:09:42.025: [    CRSD][1829238512] ENV Logging level for Module: OCRSRV  0
2011-04-27 09:09:42.027: [    CRSD][1829238512] ENV Logging level for Module: OCRMAS  0
2011-04-27 09:09:42.027: [ CRSMAIN][1829238512] Filename is /u01/crs/11.1.0/db_1/crs/init/rac02.pid
[  clsdmt][1085118784]Listening to (ADDRESS=(PROTOCOL=ipc)(KEY=rac02DBG_CRSD))
2011-04-27 09:09:42.142: [ CRSMAIN][1829238512] Using Authorizer location: /u01/crs/11.1.0/db_1/crs/auth/
2011-04-27 09:09:42.172: [ CRSMAIN][1829238512] Initializing RTI
2011-04-27 09:09:42.224: [ CRSMAIN][1829238512] Initializing EVMMgr
2011-04-27 09:09:42.224: [CRSTIMER][1358960960] Timer Thread Starting.
2011-04-27 09:09:42.391: [ COMMCRS][1367353664]clsc_connect: (0x7fbc6000a290) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 09:09:42.808: [ COMMCRS][1367353664]clsc_connect: (0x7fbc6000a290) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 09:09:43.955: [ CRSMAIN][1829238512] CRSD locked during state recovery, please wait.
2011-04-27 09:09:44.110: [ CRSMAIN][1829238512] CRSD recovered, unlocked.
2011-04-27 09:09:44.111: [ CRSMAIN][1829238512] QS socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=ora_crsqs))
2011-04-27 09:09:44.122: [ CRSMAIN][1829238512] CRSD UI socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=CRSD_UI_SOCKET))
2011-04-27 09:09:44.125: [ CRSMAIN][1829238512] E2E socket on: (ADDRESS=(PROTOCOL=tcp)(HOST=rac02-priv)(PORT=49896))
2011-04-27 09:09:44.125: [ CRSMAIN][1829238512] Starting Threads
2011-04-27 09:09:44.125: [ CRSMAIN][1829238512] CRS Daemon Started.
2011-04-27 09:09:44.125: [ CRSMAIN][1106127168] Starting runCommandServer for (UI = 1, E2E = 0). 1
2011-04-27 09:09:44.125: [ CRSMAIN][1104025920] Starting runCommandServer for (UI = 1, E2E = 0). 0
2011-04-27 09:09:44.179: [  CRSRES][1829238512]  startup = 1
2011-04-27 09:09:44.207: [  CRSRES][1829238512]  startup = 1
2011-04-27 09:09:44.253: [  CRSRES][1829238512]  startup = 1
2011-04-27 09:09:44.319: [  CRSRES][1407228224] StopResource: setting CLI values
2011-04-27 09:09:44.347: [  CRSRES][1407228224] Attempting to stop `ora.rac02.vip` on member `rac01`
2011-04-27 09:09:44.621: [  CRSRES][1407228224] Stop of `ora.rac02.vip` on member `rac01` succeeded.
2011-04-27 09:09:44.647: [  CRSRES][1407228224] startRunnable: setting CLI values
2011-04-27 09:09:44.647: [  CRSRES][1407228224] Attempting to start `ora.rac02.vip` on member `rac02`
2011-04-27 09:09:48.109: [  CRSRES][1407228224] Start of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-27 09:09:48.177: [  CRSRES][1407228224] startRunnable: setting CLI values
2011-04-27 09:09:48.188: [  CRSRES][1407228224] Attempting to start `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-27 09:09:49.305: [  CRSRES][1407228224] Start of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-27 09:09:49.366: [  CRSRES][1407228224] startRunnable: setting CLI values
2011-04-27 09:09:49.403: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 09:09:49.404: [  CRSRES][1407228224] Attempting to start `ora.rac02.gsd` on member `rac02`
2011-04-27 09:09:49.414: [  CRSRES][1075857728] Attempting to start `ora.rac02.ons` on member `rac02`
2011-04-27 09:09:49.793: [  CRSRES][1407228224] Start of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-27 09:09:50.270: [  CRSRES][1434507584] CRS-1002: Resource 'ora.rac02.LISTENER_RAC02.lsnr' is already running on member 'rac02'

2011-04-27 09:09:50.885: [  CRSRES][1075857728] Start of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-27 10:53:56.957: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 10:53:56.972: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 10:53:57.150: [    CRSD][1075857728] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 10:53:57.152: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 10:53:57.160: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 10:54:00.368: [    CRSD][1075857728] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 10:54:00.370: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 10:54:00.378: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 10:54:00.565: [    CRSD][1075857728] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 10:54:00.567: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 10:54:00.573: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 10:54:03.313: [  CRSAPP][1075857728] StartResource error for pcard.db_pcard error code = 1
2011-04-27 10:54:03.442: [    CRSD][1075857728] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 10:54:03.444: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 10:54:03.444: [  CRSRES][1075857728] StopResource: setting CLI values
2011-04-27 10:54:03.501: [    CRSD][1075857728] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 10:54:03.502: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 10:54:03.503: [  CRSRES][1075857728] StopResource: setting CLI values
2011-04-27 10:54:03.572: [    CRSD][1075857728] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 10:54:03.575: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 10:54:03.575: [  CRSRES][1075857728] StopResource: setting CLI values
2011-04-27 10:54:03.604: [    CRSD][1075857728] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:11:30.196: [  OCRMAS][1199499584]th_master:13: I AM THE NEW OCR MASTER at incar 54. Node Number 2
2011-04-27 11:11:30.225: [  OCRRAW][1199499584]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-27 11:11:30.247: [  OCRMAS][1199499584]th_master: Deleted ver keys from cache (master)
2011-04-27 11:11:41.018: [ CRSCOMM][1407228224] CLEANUP: Searching for connections to failed node rac01
2011-04-27 11:11:41.018: [  CRSEVT][1407228224] Processing member leave for rac01, incarnation: 196435626
2011-04-27 11:11:41.019: [    CRSD][1407228224] SM: recovery in process: 8
2011-04-27 11:11:41.019: [  CRSEVT][1407228224] Do failover for: rac01
2011-04-27 11:11:41.086: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:11:41.109: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:11:41.121: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:11:41.138: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:11:41.150: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:11:41.164: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:11:41.185: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:11:41.233: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:11:41.249: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:11:41.328: [  CRSRES][1434507584] startRunnable: setting CLI values
2011-04-27 11:11:41.357: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 11:11:41.375: [  CRSRES][1075857728] Attempting to start `pcard` on member `rac02`
2011-04-27 11:11:41.384: [  CRSRES][1434507584] Attempting to start `ora.rac01.vip` on member `rac02`
2011-04-27 11:11:41.401: [  CRSRES][1075857728] Start of `pcard` on member `rac02` succeeded.
2011-04-27 11:11:41.480: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 11:11:41.484: [  CRSRES][1075857728] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-27 11:11:41.997: [  CRSAPP][1075857728] StartResource error for pcard.db_pcard error code = 1
2011-04-27 11:11:42.151: [  CRSRES][1075857728] Start of `pcard.db_pcard` on member `rac02` failed.
2011-04-27 11:11:42.241: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 11:11:42.249: [  CRSRES][1075857728] Attempting to start `pcard.vip` on member `rac02`
2011-04-27 11:11:44.901: [  CRSRES][1434507584] Start of `ora.rac01.vip` on member `rac02` succeeded.
2011-04-27 11:11:45.327: [  CRSRES][1075857728] Start of `pcard.vip` on member `rac02` succeeded.
2011-04-27 11:11:45.381: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 11:11:45.383: [  CRSRES][1075857728] Attempting to start `pcard.listener` on member `rac02`
2011-04-27 11:11:45.454: [  CRSRES][1075857728] Start of `pcard.listener` on member `rac02` succeeded.
2011-04-27 11:11:45.522: [  CRSRES][1075857728] startRunnable: setting CLI values
2011-04-27 11:11:45.530: [  CRSRES][1075857728] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-27 11:11:45.992: [  CRSAPP][1075857728] StartResource error for pcard.db_pcard error code = 1
2011-04-27 11:11:46.127: [  CRSRES][1075857728] Start of `pcard.db_pcard` on member `rac02` failed.
2011-04-27 11:11:46.172: [  CRSEVT][1407228224] Post recovery done evmd event for: rac01
2011-04-27 11:11:46.172: [    CRSD][1407228224] SM: recoveryDone: 0
2011-04-27 11:11:46.186: [  CRSEVT][1407228224] Processing RecoveryDone
2011-04-27 11:14:34.902: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:14:34.902: [  CRSRES][1407228224] StopResource: setting CLI values
2011-04-27 11:14:35.169: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:14:39.504: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:14:39.504: [  CRSRES][1407228224] StopResource: setting CLI values
2011-04-27 11:14:39.576: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:14:39.620: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:14:39.620: [  CRSRES][1407228224] StopResource: setting CLI values
2011-04-27 11:14:39.688: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:14:39.740: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:14:39.741: [  CRSRES][1407228224] StopResource: setting CLI values
2011-04-27 11:14:39.757: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:16:51.355: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:16:51.381: [  CRSRES][1407228224] startRunnable: setting CLI values
2011-04-27 11:16:51.398: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:16:51.400: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:16:51.412: [  CRSRES][1407228224] startRunnable: setting CLI values
2011-04-27 11:16:54.492: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:16:54.494: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:16:54.515: [  CRSRES][1407228224] startRunnable: setting CLI values
2011-04-27 11:16:54.581: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:16:54.583: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:16:54.611: [  CRSRES][1407228224] startRunnable: setting CLI values
2011-04-27 11:16:55.072: [  CRSAPP][1407228224] StartResource error for pcard.db_pcard error code = 1
2011-04-27 11:16:55.205: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:16:55.211: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:16:55.211: [  CRSRES][1407228224] StopResource: setting CLI values
2011-04-27 11:16:55.254: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:16:55.258: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:16:55.259: [  CRSRES][1407228224] StopResource: setting CLI values
2011-04-27 11:16:55.349: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:16:55.354: [    CRSD][1104025920] SM: rE2Ec: 4
2011-04-27 11:16:55.354: [  CRSRES][1407228224] StopResource: setting CLI values
2011-04-27 11:16:55.370: [    CRSD][1407228224] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:19:30.766: [ CRSCOMM][1407228224] CLEANUP: Searching for connections to failed node rac01
2011-04-27 11:19:30.766: [  CRSEVT][1407228224] Processing member leave for rac01, incarnation: 196435628
2011-04-27 11:19:30.767: [    CRSD][1407228224] SM: recovery in process: 8
2011-04-27 11:19:30.767: [  CRSEVT][1407228224] Do failover for: rac01
2011-04-27 11:19:30.839: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:19:30.859: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:19:30.867: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:19:30.882: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:19:30.903: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:19:30.917: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:19:30.925: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:19:30.940: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:19:30.959: [  CRSRES][1407228224]  startup = 0
2011-04-27 11:19:31.038: [  CRSRES][1455495488] startRunnable: setting CLI values
2011-04-27 11:19:31.038: [  CRSRES][1455495488] Attempting to start `ora.rac01.vip` on member `rac02`
2011-04-27 11:19:31.065: [  CRSRES][1453394240] startRunnable: setting CLI values
2011-04-27 11:19:31.066: [  CRSRES][1453394240] Attempting to start `pcard` on member `rac02`
2011-04-27 11:19:31.095: [  CRSRES][1453394240] Start of `pcard` on member `rac02` succeeded.
2011-04-27 11:19:31.126: [  CRSRES][1453394240] startRunnable: setting CLI values
2011-04-27 11:19:31.127: [  CRSRES][1453394240] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-27 11:19:31.595: [  CRSAPP][1453394240] StartResource error for pcard.db_pcard error code = 1
2011-04-27 11:19:31.725: [  CRSRES][1453394240] Start of `pcard.db_pcard` on member `rac02` failed.
2011-04-27 11:19:31.788: [  CRSRES][1453394240] startRunnable: setting CLI values
2011-04-27 11:19:31.789: [  CRSRES][1453394240] Attempting to start `pcard.vip` on member `rac02`
2011-04-27 11:19:34.339: [  CRSRES][1455495488] Start of `ora.rac01.vip` on member `rac02` succeeded.
2011-04-27 11:19:34.954: [  CRSRES][1453394240] Start of `pcard.vip` on member `rac02` succeeded.
2011-04-27 11:19:35.000: [  CRSRES][1453394240] startRunnable: setting CLI values
2011-04-27 11:19:35.000: [  CRSRES][1453394240] Attempting to start `pcard.listener` on member `rac02`
2011-04-27 11:19:35.056: [  CRSRES][1453394240] Start of `pcard.listener` on member `rac02` succeeded.
2011-04-27 11:19:35.159: [  CRSRES][1453394240] startRunnable: setting CLI values
2011-04-27 11:19:35.167: [  CRSRES][1453394240] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-27 11:19:35.631: [  CRSAPP][1453394240] StartResource error for pcard.db_pcard error code = 1
2011-04-27 11:19:35.777: [  CRSRES][1453394240] Start of `pcard.db_pcard` on member `rac02` failed.
2011-04-27 11:19:35.806: [  CRSEVT][1407228224] Post recovery done evmd event for: rac01
2011-04-27 11:19:35.806: [    CRSD][1407228224] SM: recoveryDone: 0
2011-04-27 11:19:35.808: [  CRSEVT][1407228224] Processing RecoveryDone
2011-04-27 11:21:14.765: [    CRSD][1407228224] Shutdown Manager: Wait for recovery is complete.0
2011-04-27 11:21:14.765: [    CRSD][1407228224] Shutdown Manager: Wait for startup is complete
2011-04-27 11:21:14.765: [    CRSD][1407228224] Shutdown Manager: Wait for ongoing commands is complete
2011-04-27 11:21:14.774: [    CRSD][1407228224] SM: asked E2E to exit.
2011-04-27 11:21:14.774: [ CRSCOMM][1104025920] CLEANUP: quiesceing connection 0x7fbc600849d0
2011-04-27 11:21:14.774: [ CRSMAIN][1104025920] runCommandServer for E2E exiting.
2011-04-27 11:21:14.774: [    CRSD][1407228224] SM: E2E port closed.3
2011-04-27 11:21:14.774: [    CRSD][1407228224] SM: stoppedResources_all_1 m_state = 3
2011-04-27 11:21:14.783: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:14.787: [  CRSRES][1075857728] StopResource: setting CLI values
2011-04-27 11:21:14.791: [  CRSRES][1434507584] StopResource: setting CLI values
2011-04-27 11:21:14.804: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:14.832: [  CRSRES][1075857728] Attempting to stop `ora.rac02.gsd` on member `rac02`
2011-04-27 11:21:14.845: [  CRSRES][1434507584] Attempting to stop `ora.rac02.ons` on member `rac02`
2011-04-27 11:21:14.857: [  CRSRES][1104025920] Attempting to stop `pcard.listener` on member `rac02`
2011-04-27 11:21:14.883: [ CRSMAIN][1106127168] UI Cmd received after shutdown began.
2011-04-27 11:21:14.908: [  CRSRES][1104025920] Stop of `pcard.listener` on member `rac02` succeeded.
2011-04-27 11:21:14.920: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:14.947: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:14.964: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:14.971: [  CRSRES][1104025920] Attempting to stop `pcard.vip` on member `rac02`
2011-04-27 11:21:15.040: [  CRSRES][1104025920] Stop of `pcard.vip` on member `rac02` succeeded.
2011-04-27 11:21:15.055: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:15.058: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:15.069: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:15.075: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:15.080: [  CRSRES][1104025920] StopResource: setting CLI values
2011-04-27 11:21:15.080: [  CRSRES][1104025920] Attempting to stop `pcard` on member `rac02`
2011-04-27 11:21:15.096: [  CRSRES][1075857728] Stop of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-27 11:21:15.107: [  CRSRES][1104025920] Stop of `pcard` on member `rac02` succeeded.
2011-04-27 11:21:15.134: [  CRSRES][1434507584] Stop of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-27 11:21:15.140: [  CRSRES][1075857728] StopResource: setting CLI values
2011-04-27 11:21:15.146: [  CRSRES][1434507584] StopResource: setting CLI values
2011-04-27 11:21:15.147: [  CRSRES][1075857728] Attempting to stop `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-27 11:21:15.155: [  CRSRES][1434507584] StopResource: setting CLI values
2011-04-27 11:21:15.160: [  CRSRES][1434507584] Attempting to stop `ora.rac01.vip` on member `rac02`
2011-04-27 11:21:15.416: [  CRSRES][1434507584] Stop of `ora.rac01.vip` on member `rac02` succeeded.
2011-04-27 11:21:15.987: [ CRSMAIN][1106127168] UI Cmd received after shutdown began.
2011-04-27 11:21:16.118: [  CRSRES][1075857728] Stop of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-27 11:21:16.125: [  CRSRES][1075857728] StopResource: setting CLI values
2011-04-27 11:21:16.140: [  CRSRES][1075857728] StopResource: setting CLI values
2011-04-27 11:21:16.160: [  CRSRES][1075857728] Attempting to stop `ora.rac02.vip` on member `rac02`
2011-04-27 11:21:16.421: [  CRSRES][1075857728] Stop of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-27 11:21:16.505: [  CRSEVT][1403025728] Error dispatching EVM event; reconnecting
2011-04-27 11:21:16.568: [ CSSCLNT][1342175552]clssgsGGetStatus: CSS shutting down.

2011-04-27 11:21:16.568: [ CSSCLNT][1342175552]clssgsGroupGetStatus: returning 22

2011-04-27 11:21:16.568: [ CSSCLNT][1199499584]clssgsGGetStatus: CSS shutting down.

2011-04-27 11:21:16.568: [ CSSCLNT][1199499584]clssgsGroupGetStatus: returning 22

2011-04-27 11:21:16.568: [    CRSD][1342175552] CRSD exiting on CSS shutdown request
2011-04-27 11:21:16.568: [    CRSD][1342175552] Done.

2011-04-27 11:21:16.568: [  OCRMAS][1199499584]th_master:21:clssgsgrpstat failed css ret code = 22
2011-04-27 11:21:48.484: [ default][2612627184] CRS Daemon Starting
2011-04-27 11:21:48.484: [ CRSMAIN][2612627184] Checking the OCR device
2011-04-27 11:21:48.535: [ CRSMAIN][2612627184] Connecting to the CSS Daemon
2011-04-27 11:21:48.703: [ COMMCRS][1112488256]clsc_connect: (0x170d3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 11:21:48.703: [ CSSCLNT][2612627184]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 11:21:48.703: [  CRSRTI][2612627184] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 11:21:49.869: [ COMMCRS][1112488256]clsc_connect: (0x170d3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 11:21:49.869: [ CSSCLNT][2612627184]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 11:21:49.869: [  CRSRTI][2612627184] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 11:21:51.036: [ COMMCRS][1112488256]clsc_connect: (0x170d3d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 11:21:51.036: [ CSSCLNT][2612627184]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 11:21:51.036: [  CRSRTI][2612627184] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 11:22:01.002: [ CRSMAIN][2612627184] CRSD running as the Privileged user

2011-04-27 11:22:01.240: [  CLSVER][2612627184] Active Version from OCR:11.1.0.7.0
2011-04-27 11:22:01.240: [  CLSVER][2612627184] Active Version and Software Version are same
2011-04-27 11:22:01.240: [ CRSMAIN][2612627184] Initializing OCR
2011-04-27 11:22:01.300: [  OCRRAW][2612627184]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-27 11:22:01.362: [  OCRMAS][1188022592]th_master:12: I AM THE NEW OCR MASTER at incar 1. Node Number 2
2011-04-27 11:22:01.398: [  OCRRAW][1188022592]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-27 11:22:01.458: [    CRSD][2612627184] ENV Logging level for Module: allcomp  0
2011-04-27 11:22:01.460: [    CRSD][2612627184] ENV Logging level for Module: default  0
2011-04-27 11:22:01.461: [    CRSD][2612627184] ENV Logging level for Module: COMMCRS  0
2011-04-27 11:22:01.462: [    CRSD][2612627184] ENV Logging level for Module: COMMNS  0
2011-04-27 11:22:01.464: [    CRSD][2612627184] ENV Logging level for Module: CRSUI  0
2011-04-27 11:22:01.465: [  OCRMAS][1188022592]th_master: Deleted ver keys from cache (master)
2011-04-27 11:22:01.466: [    CRSD][2612627184] ENV Logging level for Module: CRSCOMM  0
2011-04-27 11:22:01.467: [    CRSD][2612627184] ENV Logging level for Module: CRSRTI  0
2011-04-27 11:22:01.468: [    CRSD][2612627184] ENV Logging level for Module: CRSMAIN  0
2011-04-27 11:22:01.480: [    CRSD][2612627184] ENV Logging level for Module: CRSPLACE  0
2011-04-27 11:22:01.482: [    CRSD][2612627184] ENV Logging level for Module: CRSAPP  0
2011-04-27 11:22:01.483: [    CRSD][2612627184] ENV Logging level for Module: CRSRES  0
2011-04-27 11:22:01.484: [    CRSD][2612627184] ENV Logging level for Module: CRSOCR  0
2011-04-27 11:22:01.487: [    CRSD][2612627184] ENV Logging level for Module: CRSTIMER  0
2011-04-27 11:22:01.487: [    CRSD][2612627184] ENV Logging level for Module: CRSEVT  0
2011-04-27 11:22:01.488: [    CRSD][2612627184] ENV Logging level for Module: CRSD  0
2011-04-27 11:22:01.490: [    CRSD][2612627184] ENV Logging level for Module: CLUCLS  0
2011-04-27 11:22:01.492: [    CRSD][2612627184] ENV Logging level for Module: CLSVER  0
2011-04-27 11:22:01.494: [    CRSD][2612627184] ENV Logging level for Module: OCRRAW  0
2011-04-27 11:22:01.496: [    CRSD][2612627184] ENV Logging level for Module: OCROSD  0
2011-04-27 11:22:01.497: [    CRSD][2612627184] ENV Logging level for Module: OCRCAC  0
2011-04-27 11:22:01.498: [    CRSD][2612627184] ENV Logging level for Module: CSSCLNT  0
2011-04-27 11:22:01.499: [    CRSD][2612627184] ENV Logging level for Module: OCRAPI  0
2011-04-27 11:22:01.499: [    CRSD][2612627184] ENV Logging level for Module: OCRUTL  0
2011-04-27 11:22:01.500: [    CRSD][2612627184] ENV Logging level for Module: OCRMSG  0
2011-04-27 11:22:01.501: [    CRSD][2612627184] ENV Logging level for Module: OCRCLI  0
2011-04-27 11:22:01.501: [    CRSD][2612627184] ENV Logging level for Module: OCRSRV  0
2011-04-27 11:22:01.502: [    CRSD][2612627184] ENV Logging level for Module: OCRMAS  0
2011-04-27 11:22:01.502: [ CRSMAIN][2612627184] Filename is /u01/crs/11.1.0/db_1/crs/init/rac02.pid
[  clsdmt][1355876672]Listening to (ADDRESS=(PROTOCOL=ipc)(KEY=rac02DBG_CRSD))
2011-04-27 11:22:01.511: [  OCRUTL][2612627184]u_freem: mem passed is null
2011-04-27 11:22:01.520: [ CRSMAIN][2612627184] Using Authorizer location: /u01/crs/11.1.0/db_1/crs/auth/
2011-04-27 11:22:01.565: [ CRSMAIN][2612627184] Initializing RTI
2011-04-27 11:22:01.606: [  OCRUTL][2612627184]u_freem: mem passed is null
2011-04-27 11:22:01.646: [ CRSMAIN][2612627184] Initializing EVMMgr
2011-04-27 11:22:01.646: [CRSTIMER][1372662080] Timer Thread Starting.
2011-04-27 11:22:01.813: [ COMMCRS][1381054784]clsc_connect: (0x7f669402dd70) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:22:03.049: [ CRSMAIN][2612627184] CRSD locked during state recovery, please wait.
2011-04-27 11:22:03.237: [ CRSMAIN][2612627184] CRSD recovered, unlocked.
2011-04-27 11:22:03.238: [ CRSMAIN][2612627184] QS socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=ora_crsqs))
2011-04-27 11:22:03.245: [ CRSMAIN][2612627184] CRSD UI socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=CRSD_UI_SOCKET))
2011-04-27 11:22:03.248: [ CRSMAIN][2612627184] E2E socket on: (ADDRESS=(PROTOCOL=tcp)(HOST=rac02-priv)(PORT=49896))
2011-04-27 11:22:03.248: [ CRSMAIN][2612627184] Starting Threads
2011-04-27 11:22:03.248: [ CRSMAIN][1416726848] Starting runCommandServer for (UI = 1, E2E = 0). 0
2011-04-27 11:22:03.248: [ CRSMAIN][2612627184] CRS Daemon Started.
2011-04-27 11:22:03.248: [ CRSMAIN][1418828096] Starting runCommandServer for (UI = 1, E2E = 0). 1
2011-04-27 11:22:03.258: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.283: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.308: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.318: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.328: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.338: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.349: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.362: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.374: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.383: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.392: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.402: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.415: [  CRSRES][2612627184]  startup = 1
2011-04-27 11:22:03.469: [  CRSRES][1423030592] startRunnable: setting CLI values
2011-04-27 11:22:03.475: [  CRSRES][1423030592] Attempting to start `ora.rac02.vip` on member `rac02`
2011-04-27 11:22:06.853: [  CRSRES][1423030592] Start of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-27 11:22:06.937: [  CRSRES][1423030592] startRunnable: setting CLI values
2011-04-27 11:22:06.964: [  CRSRES][1423030592] Attempting to start `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-27 11:22:07.468: [  CRSRES][1423030592] Start of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-27 11:22:07.503: [  CRSRES][1433524544] startRunnable: setting CLI values
2011-04-27 11:22:07.530: [  CRSRES][1423030592] startRunnable: setting CLI values
2011-04-27 11:22:07.543: [  CRSRES][1448220992] startRunnable: setting CLI values
2011-04-27 11:22:07.576: [  CRSRES][1433524544] Attempting to start `ora.rac01.vip` on member `rac02`
2011-04-27 11:22:07.584: [  CRSRES][1450322240] startRunnable: setting CLI values
2011-04-27 11:22:07.596: [  CRSRES][1423030592] Attempting to start `pcard` on member `rac02`
2011-04-27 11:22:07.604: [  CRSRES][1448220992] Attempting to start `ora.rac02.gsd` on member `rac02`
2011-04-27 11:22:07.621: [  CRSRES][1450322240] Attempting to start `ora.rac02.ons` on member `rac02`
2011-04-27 11:22:08.012: [  CRSRES][1423030592] Start of `pcard` on member `rac02` succeeded.
2011-04-27 11:22:08.064: [  CRSRES][1423030592] startRunnable: setting CLI values
2011-04-27 11:22:08.108: [  CRSRES][1423030592] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-27 11:22:08.110: [  CRSRES][1454524736] CRS-1002: Resource 'ora.rac02.LISTENER_RAC02.lsnr' is already running on member 'rac02'

2011-04-27 11:22:08.380: [  CRSRES][1448220992] Start of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-27 11:22:08.682: [  CRSAPP][1423030592] StartResource error for pcard.db_pcard error code = 1
2011-04-27 11:22:08.823: [  CRSRES][1423030592] Start of `pcard.db_pcard` on member `rac02` failed.
2011-04-27 11:22:08.900: [  CRSRES][1423030592] startRunnable: setting CLI values
2011-04-27 11:22:08.937: [  CRSRES][1423030592] Attempting to start `pcard.vip` on member `rac02`
2011-04-27 11:22:09.436: [  CRSRES][1450322240] Start of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-27 11:22:11.330: [  CRSRES][1433524544] Start of `ora.rac01.vip` on member `rac02` succeeded.
2011-04-27 11:22:12.110: [  CRSRES][1423030592] Start of `pcard.vip` on member `rac02` succeeded.
2011-04-27 11:22:12.153: [  CRSRES][1423030592] startRunnable: setting CLI values
2011-04-27 11:22:12.191: [  CRSRES][1423030592] Attempting to start `pcard.listener` on member `rac02`
2011-04-27 11:22:12.343: [  CRSRES][1423030592] Start of `pcard.listener` on member `rac02` succeeded.
2011-04-27 11:22:12.402: [  CRSRES][1423030592] startRunnable: setting CLI values
2011-04-27 11:22:12.410: [  CRSRES][1423030592] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-27 11:22:12.874: [  CRSAPP][1423030592] StartResource error for pcard.db_pcard error code = 1
2011-04-27 11:22:13.001: [  CRSRES][1423030592] Start of `pcard.db_pcard` on member `rac02` failed.
2011-04-27 11:24:25.982: [  CRSEVT][1420929344] Error dispatching EVM event; reconnecting
2011-04-27 11:24:26.149: [ COMMCRS][1381054784]clsc_connect: (0x215a5e0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:27.066: [ COMMCRS][1381054784]clsc_connect: (0x2147ca0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:27.732: [ COMMCRS][1381054784]clsc_connect: (0x2159bb0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:27.954: [    CRSD][1450322240] Shutdown Manager: Wait for recovery is complete.0
2011-04-27 11:24:27.954: [    CRSD][1450322240] Shutdown Manager: Wait for startup is complete
2011-04-27 11:24:27.954: [    CRSD][1450322240] Shutdown Manager: Wait for ongoing commands is complete
2011-04-27 11:24:27.964: [    CRSD][1450322240] SM: asked E2E to exit.
2011-04-27 11:24:27.964: [ CRSCOMM][1416726848] CLEANUP: quiesceing connection 0x7f66940a1aa0
2011-04-27 11:24:27.964: [ CRSMAIN][1416726848] runCommandServer for E2E exiting.
2011-04-27 11:24:27.964: [    CRSD][1450322240] SM: E2E port closed.3
2011-04-27 11:24:27.964: [    CRSD][1450322240] SM: stoppedResources_all_1 m_state = 3
2011-04-27 11:24:27.973: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:27.979: [  CRSRES][1448220992] StopResource: setting CLI values
2011-04-27 11:24:27.984: [  CRSRES][1446119744] StopResource: setting CLI values
2011-04-27 11:24:28.002: [  CRSRES][1448220992] Attempting to stop `ora.rac02.gsd` on member `rac02`
2011-04-27 11:24:28.002: [ COMMCRS][1448220992]clscsendx: (0x2119960) Physical connection (0x2119200) not active

2011-04-27 11:24:28.021: [  CRSRES][1446119744] Attempting to stop `ora.rac02.ons` on member `rac02`
2011-04-27 11:24:28.106: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:28.116: [  CRSRES][1416726848] Attempting to stop `pcard.listener` on member `rac02`
2011-04-27 11:24:28.187: [ COMMCRS][1431423296]clsc_connect: (0x7f668c005ea0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:28.188: [  CRSEVT][1446119744] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:28.282: [ COMMCRS][1381054784]clsc_connect: (0x2147ef0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:28.283: [  CRSEVT][1416726848] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:28.522: [  CRSRES][1448220992] Stop of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-27 11:24:28.626: [  CRSRES][1446119744] Stop of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-27 11:24:28.690: [ COMMCRS][1381054784]clsc_connect: (0x7f66940cf040) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:28.690: [  CRSEVT][1448220992] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:28.793: [ COMMCRS][1431423296]clsc_connect: (0x7f668c02a220) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:28.793: [  CRSEVT][1446119744] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:29.150: [ COMMCRS][1431423296]clsc_connect: (0x215a5e0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:29.296: [ CRSMAIN][1418828096] UI Cmd received after shutdown began.
2011-04-27 11:24:29.315: [  CRSRES][1416726848] Stop of `pcard.listener` on member `rac02` succeeded.
2011-04-27 11:24:29.482: [ COMMCRS][1431423296]clsc_connect: (0x21510e0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:29.482: [  CRSEVT][1416726848] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:29.493: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:29.501: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:29.519: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:29.525: [  CRSRES][1416726848] Attempting to stop `pcard.vip` on member `rac02`
2011-04-27 11:24:29.692: [ COMMCRS][1431423296]clsc_connect: (0x2153e30) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:29.692: [  CRSEVT][1416726848] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:29.762: [  CRSRES][1416726848] Stop of `pcard.vip` on member `rac02` succeeded.
2011-04-27 11:24:29.929: [ COMMCRS][1431423296]clsc_connect: (0x2154080) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:29.930: [  CRSEVT][1416726848] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:29.946: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:29.953: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:29.981: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:29.988: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:30.008: [  CRSRES][1416726848] StopResource: setting CLI values
2011-04-27 11:24:30.015: [  CRSRES][1416726848] Attempting to stop `pcard` on member `rac02`
2011-04-27 11:24:30.067: [ COMMCRS][1381054784]clsc_connect: (0x211f160) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:30.182: [ COMMCRS][1431423296]clsc_connect: (0x2158bd0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:30.182: [  CRSEVT][1416726848] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:30.203: [  CRSRES][1416726848] Stop of `pcard` on member `rac02` succeeded.
2011-04-27 11:24:30.370: [ COMMCRS][1431423296]clsc_connect: (0x215a2c0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:30.370: [  CRSEVT][1416726848] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:30.379: [  CRSRES][1446119744] StopResource: setting CLI values
2011-04-27 11:24:30.386: [  CRSRES][1448220992] StopResource: setting CLI values
2011-04-27 11:24:30.393: [  CRSRES][1446119744] Attempting to stop `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-27 11:24:30.430: [  CRSRES][1448220992] StopResource: setting CLI values
2011-04-27 11:24:30.435: [  CRSRES][1448220992] Attempting to stop `ora.rac01.vip` on member `rac02`
2011-04-27 11:24:30.560: [ COMMCRS][1431423296]clsc_connect: (0x7f668c0067a0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:30.561: [  CRSEVT][1446119744] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:30.603: [ COMMCRS][1381054784]clsc_connect: (0x2147ef0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:30.603: [  CRSEVT][1448220992] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:30.734: [ COMMCRS][1431423296]clsc_connect: (0x2159060) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:31.035: [  CRSRES][1448220992] Stop of `ora.rac01.vip` on member `rac02` succeeded.
2011-04-27 11:24:31.202: [ COMMCRS][1431423296]clsc_connect: (0x215a5e0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:31.203: [  CRSEVT][1448220992] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:31.297: [ CRSMAIN][1418828096] UI Cmd received after shutdown began.
2011-04-27 11:24:31.604: [  CRSRES][1446119744] Stop of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-27 11:24:31.771: [ COMMCRS][1431423296]clsc_connect: (0x7f66940cffe0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:31.772: [  CRSEVT][1446119744] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:31.780: [  CRSRES][1446119744] StopResource: setting CLI values
2011-04-27 11:24:31.816: [  CRSRES][1446119744] StopResource: setting CLI values
2011-04-27 11:24:31.836: [  CRSRES][1446119744] Attempting to stop `ora.rac02.vip` on member `rac02`
2011-04-27 11:24:31.902: [ COMMCRS][1381054784]clsc_connect: (0x2154df0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:32.002: [ COMMCRS][1431423296]clsc_connect: (0x7f66940cffe0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:32.002: [  CRSEVT][1446119744] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:32.643: [  CRSRES][1446119744] Stop of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-27 11:24:32.810: [ COMMCRS][1431423296]clsc_connect: (0x7f66940cedf0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:24:32.810: [  CRSEVT][1446119744] postEvent, Could not connect to EVMD; functionality will be limited

2011-04-27 11:24:32.984: [ CSSCLNT][1188022592]clssgsGGetStatus: CSS shutting down.

2011-04-27 11:24:32.984: [ CSSCLNT][1188022592]clssgsGroupGetStatus: returning 22

2011-04-27 11:24:32.985: [  OCRMAS][1188022592]th_master:21:clssgsgrpstat failed css ret code = 22
[  OCRAPI][1188022592]procr_ctx_set_invalid_no_abort: ctx set to invalid
2011-04-27 11:24:33.008: [ CSSCLNT][1347483968]clssgsGGetStatus: CSS shutting down.

2011-04-27 11:24:33.008: [ CSSCLNT][1347483968]clssgsGroupGetStatus: returning 22

2011-04-27 11:24:33.008: [    CRSD][1347483968] CRSD exiting on CSS shutdown request
2011-04-27 11:24:33.008: [    CRSD][1347483968] Done.

2011-04-27 11:29:16.212: [ default][2742634224] CRS Daemon Starting
2011-04-27 11:29:16.229: [ CRSMAIN][2742634224] Checking the OCR device
2011-04-27 11:29:16.251: [ CRSMAIN][2742634224] Connecting to the CSS Daemon
2011-04-27 11:29:16.418: [ COMMCRS][1104537920]clsc_connect: (0xe023d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 11:29:16.419: [ CSSCLNT][2742634224]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 11:29:16.419: [  CRSRTI][2742634224] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 11:29:17.585: [ COMMCRS][1104537920]clsc_connect: (0xe023d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 11:29:17.586: [ CSSCLNT][2742634224]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 11:29:17.586: [  CRSRTI][2742634224] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 11:29:18.754: [ COMMCRS][1104537920]clsc_connect: (0xe023d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 11:29:18.754: [ CSSCLNT][2742634224]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 11:29:18.754: [  CRSRTI][2742634224] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 11:29:19.921: [ COMMCRS][1104537920]clsc_connect: (0xe023d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-27 11:29:19.921: [ CSSCLNT][2742634224]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-27 11:29:19.921: [  CRSRTI][2742634224] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-27 11:29:20.992: [ CRSMAIN][2742634224] CRSD running as the Privileged user

2011-04-27 11:29:21.201: [  CLSVER][2742634224] Active Version from OCR:11.1.0.7.0
2011-04-27 11:29:21.201: [  CLSVER][2742634224] Active Version and Software Version are same
2011-04-27 11:29:21.201: [ CRSMAIN][2742634224] Initializing OCR
2011-04-27 11:29:21.241: [  OCRRAW][2742634224]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-27 11:29:21.288: [    CRSD][2742634224] ENV Logging level for Module: allcomp  0
2011-04-27 11:29:21.290: [    CRSD][2742634224] ENV Logging level for Module: default  0
2011-04-27 11:29:21.291: [    CRSD][2742634224] ENV Logging level for Module: COMMCRS  0
2011-04-27 11:29:21.294: [    CRSD][2742634224] ENV Logging level for Module: COMMNS  0
2011-04-27 11:29:21.296: [    CRSD][2742634224] ENV Logging level for Module: CRSUI  0
2011-04-27 11:29:21.298: [    CRSD][2742634224] ENV Logging level for Module: CRSCOMM  0
2011-04-27 11:29:21.308: [    CRSD][2742634224] ENV Logging level for Module: CRSRTI  0
2011-04-27 11:29:21.313: [    CRSD][2742634224] ENV Logging level for Module: CRSMAIN  0
2011-04-27 11:29:21.316: [    CRSD][2742634224] ENV Logging level for Module: CRSPLACE  0
2011-04-27 11:29:21.318: [    CRSD][2742634224] ENV Logging level for Module: CRSAPP  0
2011-04-27 11:29:21.320: [    CRSD][2742634224] ENV Logging level for Module: CRSRES  0
2011-04-27 11:29:21.322: [    CRSD][2742634224] ENV Logging level for Module: CRSOCR  0
2011-04-27 11:29:21.326: [    CRSD][2742634224] ENV Logging level for Module: CRSTIMER  0
2011-04-27 11:29:21.328: [    CRSD][2742634224] ENV Logging level for Module: CRSEVT  0
2011-04-27 11:29:21.330: [    CRSD][2742634224] ENV Logging level for Module: CRSD  0
2011-04-27 11:29:21.332: [    CRSD][2742634224] ENV Logging level for Module: CLUCLS  0
2011-04-27 11:29:21.334: [    CRSD][2742634224] ENV Logging level for Module: CLSVER  0
2011-04-27 11:29:21.337: [    CRSD][2742634224] ENV Logging level for Module: OCRRAW  0
2011-04-27 11:29:21.339: [    CRSD][2742634224] ENV Logging level for Module: OCROSD  0
2011-04-27 11:29:21.341: [    CRSD][2742634224] ENV Logging level for Module: OCRCAC  0
2011-04-27 11:29:21.346: [    CRSD][2742634224] ENV Logging level for Module: CSSCLNT  0
2011-04-27 11:29:21.348: [    CRSD][2742634224] ENV Logging level for Module: OCRAPI  0
2011-04-27 11:29:21.350: [    CRSD][2742634224] ENV Logging level for Module: OCRUTL  0
2011-04-27 11:29:21.353: [    CRSD][2742634224] ENV Logging level for Module: OCRMSG  0
2011-04-27 11:29:21.355: [    CRSD][2742634224] ENV Logging level for Module: OCRCLI  0
2011-04-27 11:29:21.357: [    CRSD][2742634224] ENV Logging level for Module: OCRSRV  0
2011-04-27 11:29:21.360: [    CRSD][2742634224] ENV Logging level for Module: OCRMAS  0
2011-04-27 11:29:21.360: [ CRSMAIN][2742634224] Filename is /u01/crs/11.1.0/db_1/crs/init/rac02.pid
[  clsdmt][1339533632]Listening to (ADDRESS=(PROTOCOL=ipc)(KEY=rac02DBG_CRSD))
2011-04-27 11:29:21.429: [ CRSMAIN][2742634224] Using Authorizer location: /u01/crs/11.1.0/db_1/crs/auth/
2011-04-27 11:29:21.458: [ CRSMAIN][2742634224] Initializing RTI
2011-04-27 11:29:21.497: [ CRSMAIN][2742634224] Initializing EVMMgr
2011-04-27 11:29:21.497: [CRSTIMER][1356319040] Timer Thread Starting.
2011-04-27 11:29:21.663: [ COMMCRS][1364711744]clsc_connect: (0x7f389401b120) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-04-27 11:29:22.277: [ CRSMAIN][2742634224] CRSD locked during state recovery, please wait.
2011-04-27 11:29:22.418: [ CRSMAIN][2742634224] CRSD recovered, unlocked.
2011-04-27 11:29:22.420: [ CRSMAIN][2742634224] QS socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=ora_crsqs))
2011-04-27 11:29:22.431: [ CRSMAIN][2742634224] CRSD UI socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=CRSD_UI_SOCKET))
2011-04-27 11:29:22.444: [ CRSMAIN][2742634224] E2E socket on: (ADDRESS=(PROTOCOL=tcp)(HOST=rac02-priv)(PORT=49896))
2011-04-27 11:29:22.445: [ CRSMAIN][2742634224] Starting Threads
2011-04-27 11:29:22.445: [ CRSMAIN][1400383808] Starting runCommandServer for (UI = 1, E2E = 0). 0
2011-04-27 11:29:22.445: [ CRSMAIN][1402485056] Starting runCommandServer for (UI = 1, E2E = 0). 1
2011-04-27 11:29:22.445: [ CRSMAIN][2742634224] CRS Daemon Started.
2011-04-27 11:29:22.513: [  CRSRES][2742634224]  startup = 1
2011-04-27 11:29:22.537: [  CRSRES][2742634224]  startup = 1
2011-04-27 11:29:22.586: [  CRSRES][2742634224]  startup = 1
2011-04-27 11:29:22.659: [  CRSRES][1406687552] StopResource: setting CLI values
2011-04-27 11:29:22.678: [  CRSRES][1406687552] Attempting to stop `ora.rac02.vip` on member `rac01`
2011-04-27 11:29:22.968: [  CRSRES][1406687552] Stop of `ora.rac02.vip` on member `rac01` succeeded.
2011-04-27 11:29:22.985: [  CRSRES][1406687552] startRunnable: setting CLI values
2011-04-27 11:29:22.986: [  CRSRES][1406687552] Attempting to start `ora.rac02.vip` on member `rac02`
2011-04-27 11:29:26.455: [  CRSRES][1406687552] Start of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-27 11:29:26.550: [  CRSRES][1406687552] startRunnable: setting CLI values
2011-04-27 11:29:26.613: [  CRSRES][1406687552] Attempting to start `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-27 11:29:27.861: [  CRSRES][1406687552] Start of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-27 11:29:27.932: [  CRSRES][1406687552] startRunnable: setting CLI values
2011-04-27 11:29:28.000: [  CRSRES][1406687552] Attempting to start `ora.rac02.gsd` on member `rac02`
2011-04-27 11:29:28.002: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-27 11:29:28.033: [  CRSRES][1433966912] Attempting to start `ora.rac02.ons` on member `rac02`
2011-04-27 11:29:28.609: [  CRSRES][1406687552] Start of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-27 11:29:28.660: [  CRSRES][1436068160] CRS-1002: Resource 'ora.rac02.LISTENER_RAC02.lsnr' is already running on member 'rac02'

2011-04-27 11:29:29.758: [  CRSRES][1433966912] Start of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-27 11:34:36.897: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-27 11:34:36.909: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-27 11:34:36.988: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:34:36.991: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-27 11:34:37.003: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-27 11:34:40.145: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:34:40.148: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-27 11:34:40.164: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-27 11:34:40.308: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:34:40.310: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-27 11:34:40.385: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-27 11:35:09.359: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-27 11:35:09.362: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-27 11:35:09.380: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-27 11:35:09.452: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-28 10:07:27.088: [  CRSAPP][1433966912] CheckResource error for pcard.db_pcard error code = 1
2011-04-28 10:07:27.092: [  CRSRES][1433966912] In stateChanged, pcard.db_pcard target is ONLINE
2011-04-28 10:07:27.092: [  CRSRES][1433966912] pcard.db_pcard on rac02 went OFFLINE unexpectedly
2011-04-28 10:07:27.092: [  CRSRES][1433966912] StopResource: setting CLI values
2011-04-28 10:07:27.115: [  CRSRES][1433966912] Attempting to stop `pcard.db_pcard` on member `rac02`
2011-04-28 10:07:27.306: [  CRSRES][1433966912] Stop of `pcard.db_pcard` on member `rac02` succeeded.
2011-04-28 10:07:27.306: [  CRSRES][1433966912] pcard.db_pcard RESTART_COUNT=0 RESTART_ATTEMPTS=5
2011-04-28 10:07:27.307: [  CRSRES][1433966912] Restarting pcard.db_pcard on rac02
2011-04-28 10:07:27.317: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-28 10:07:27.317: [  CRSRES][1433966912] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-28 10:07:27.827: [  CRSAPP][1433966912] StartResource error for pcard.db_pcard error code = 1
2011-04-28 10:07:28.010: [  CRSRES][1433966912] Start of `pcard.db_pcard` on member `rac02` failed.
2011-04-28 10:07:28.038: [  CRSRES][1433966912] pcard.db_pcard failed on rac02 relocating.
2011-04-28 10:07:28.258: [  CRSRES][1433966912] StopResource: setting CLI values
2011-04-28 10:07:28.288: [  CRSRES][1433966912] Attempting to stop `pcard.head` on member `rac02`
2011-04-28 10:07:28.307: [  CRSRES][1433966912] Stop of `pcard.head` on member `rac02` succeeded.
2011-04-28 10:07:28.308: [  CRSRES][1433966912] StopResource: setting CLI values
2011-04-28 10:07:28.319: [  CRSRES][1433966912] Attempting to stop `pcard.listener` on member `rac02`
2011-04-28 10:07:28.366: [  CRSRES][1433966912] Stop of `pcard.listener` on member `rac02` succeeded.
2011-04-28 10:07:28.366: [  CRSRES][1433966912] StopResource: setting CLI values
2011-04-28 10:07:28.374: [  CRSRES][1433966912] Attempting to stop `pcard.vip` on member `rac02`
2011-04-28 10:07:28.442: [  CRSRES][1433966912] Stop of `pcard.vip` on member `rac02` succeeded.
2011-04-28 10:07:28.443: [  CRSRES][1433966912] StopResource: setting CLI values
2011-04-28 10:07:28.457: [  CRSRES][1433966912] Attempting to stop `pcard` on member `rac02`
2011-04-28 10:07:28.484: [  CRSRES][1433966912] Stop of `pcard` on member `rac02` succeeded.
2011-04-28 10:07:28.507: [  CRSRES][1433966912] Attempting to start `pcard` on member `rac01`
2011-04-28 10:07:28.525: [  CRSRES][1433966912] Start of `pcard` on member `rac01` succeeded.
2011-04-28 10:07:28.698: [  CRSRES][1433966912] Attempting to start `pcard.vip` on member `rac01`
2011-04-28 10:07:31.782: [  CRSRES][1433966912] Start of `pcard.vip` on member `rac01` succeeded.
2011-04-28 10:07:31.848: [  CRSRES][1433966912] Attempting to start `pcard.listener` on member `rac01`
2011-04-28 10:07:31.916: [  CRSRES][1433966912] Start of `pcard.listener` on member `rac01` succeeded.
2011-04-28 10:07:31.974: [  CRSRES][1433966912] Attempting to start `pcard.db_pcard` on member `rac01`
2011-04-28 10:08:00.577: [  CRSRES][1433966912] Start of `pcard.db_pcard` on member `rac01` succeeded.
2011-04-28 10:08:00.586: [  CRSRES][1433966912] Attempting to start `pcard.head` on member `rac01`
2011-04-28 10:08:00.604: [  CRSRES][1433966912] Start of `pcard.head` on member `rac01` succeeded.
2011-04-28 10:11:34.535: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-28 10:11:34.558: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-28 10:11:34.584: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-28 10:11:34.586: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-28 10:11:34.600: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-28 10:11:37.685: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-28 10:11:37.687: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-28 10:11:37.712: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-28 10:11:37.796: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-28 10:11:37.798: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-28 10:11:37.843: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-28 10:12:01.860: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-28 10:12:01.862: [    CRSD][1400383808] SM: rE2Ec: 4
2011-04-28 10:12:01.906: [  CRSRES][1433966912] startRunnable: setting CLI values
2011-04-28 10:12:01.947: [    CRSD][1433966912] SM:dE2Ec: all E2E cmds done. 0
2011-04-28 10:17:55.219: [  CRSRES][1436068160] StopResource: setting CLI values
2011-04-28 10:17:55.243: [  CRSRES][1436068160] Attempting to stop `pcard.head` on member `rac02`
2011-04-28 10:17:55.263: [  CRSRES][1436068160] Stop of `pcard.head` on member `rac02` succeeded.
2011-04-28 10:17:55.263: [  CRSRES][1436068160] StopResource: setting CLI values
2011-04-28 10:17:55.277: [  CRSRES][1436068160] Attempting to stop `pcard.db_pcard` on member `rac02`
2011-04-28 10:18:20.548: [  CRSRES][1436068160] Stop of `pcard.db_pcard` on member `rac02` succeeded.
2011-04-28 10:18:20.549: [  CRSRES][1436068160] StopResource: setting CLI values
2011-04-28 10:18:20.624: [  CRSRES][1436068160] Attempting to stop `pcard.listener` on member `rac02`
2011-04-28 10:18:20.673: [  CRSRES][1436068160] Stop of `pcard.listener` on member `rac02` succeeded.
2011-04-28 10:18:20.674: [  CRSRES][1436068160] StopResource: setting CLI values
2011-04-28 10:18:20.687: [  CRSRES][1436068160] Attempting to stop `pcard.vip` on member `rac02`
2011-04-28 10:18:20.767: [  CRSRES][1436068160] Stop of `pcard.vip` on member `rac02` succeeded.
2011-04-28 10:18:20.768: [  CRSRES][1436068160] StopResource: setting CLI values
2011-04-28 10:18:20.768: [  CRSRES][1436068160] Attempting to stop `pcard` on member `rac02`
2011-04-28 10:18:20.793: [  CRSRES][1436068160] Stop of `pcard` on member `rac02` succeeded.
2011-04-28 10:18:20.811: [  CRSRES][1436068160] Attempting to start `pcard` on member `rac01`
2011-04-28 10:18:20.843: [  CRSRES][1436068160] Start of `pcard` on member `rac01` succeeded.
2011-04-28 10:18:20.868: [  CRSRES][1436068160] Attempting to start `pcard.vip` on member `rac01`
2011-04-28 10:18:23.949: [  CRSRES][1436068160] Start of `pcard.vip` on member `rac01` succeeded.
2011-04-28 10:18:23.983: [  CRSRES][1436068160] Attempting to start `pcard.listener` on member `rac01`
2011-04-28 10:18:24.046: [  CRSRES][1436068160] Start of `pcard.listener` on member `rac01` succeeded.
2011-04-28 10:18:24.089: [  CRSRES][1436068160] Attempting to start `pcard.db_pcard` on member `rac01`
2011-04-28 10:18:51.753: [  CRSRES][1436068160] Start of `pcard.db_pcard` on member `rac01` succeeded.
2011-04-28 10:18:51.777: [  CRSRES][1436068160] Attempting to start `pcard.head` on member `rac01`
2011-04-28 10:18:51.813: [  CRSRES][1436068160] Start of `pcard.head` on member `rac01` succeeded.
2011-04-29 03:10:02.128: [    CRSD][1433966912] Shutdown Manager: Wait for recovery is complete.0
2011-04-29 03:10:02.128: [    CRSD][1433966912] Shutdown Manager: Wait for startup is complete
2011-04-29 03:10:02.128: [    CRSD][1433966912] Shutdown Manager: Wait for ongoing commands is complete
2011-04-29 03:10:02.139: [    CRSD][1433966912] SM: asked E2E to exit.
2011-04-29 03:10:02.139: [ CRSCOMM][1400383808] CLEANUP: quiesceing connection 0x7f389409d760
2011-04-29 03:10:02.140: [ CRSMAIN][1400383808] runCommandServer for E2E exiting.
2011-04-29 03:10:02.140: [    CRSD][1433966912] SM: E2E port closed.3
2011-04-29 03:10:02.140: [    CRSD][1433966912] SM: stoppedResources_all_1 m_state = 3
2011-04-29 03:10:02.152: [  CRSRES][1400383808] StopResource: setting CLI values
2011-04-29 03:10:02.163: [  CRSRES][1436068160] StopResource: setting CLI values
2011-04-29 03:10:02.171: [  CRSRES][1400383808] Attempting to stop `ora.rac02.gsd` on member `rac02`
2011-04-29 03:10:02.238: [  CRSRES][1436068160] Attempting to stop `ora.rac02.ons` on member `rac02`
2011-04-29 03:10:02.453: [  CRSRES][1400383808] Stop of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-29 03:10:02.517: [  CRSRES][1436068160] Stop of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-29 03:10:02.529: [  CRSRES][1436068160] StopResource: setting CLI values
2011-04-29 03:10:02.544: [  CRSRES][1436068160] Attempting to stop `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-29 03:10:03.346: [ CRSMAIN][1402485056] UI Cmd received after shutdown began.
2011-04-29 03:10:03.527: [  CRSRES][1436068160] Stop of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-29 03:10:03.545: [  CRSRES][1436068160] StopResource: setting CLI values
2011-04-29 03:10:03.574: [  CRSRES][1436068160] StopResource: setting CLI values
2011-04-29 03:10:03.593: [  CRSRES][1436068160] Attempting to stop `ora.rac02.vip` on member `rac02`
2011-04-29 03:10:03.870: [  CRSRES][1436068160] Stop of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-29 03:10:04.043: [  CRSEVT][1404586304] Error dispatching EVM event; reconnecting
2011-04-29 03:10:04.055: [ CSSCLNT][1331140928]clssgsGGetStatus: CSS shutting down.

2011-04-29 03:10:04.055: [ CSSCLNT][1331140928]clssgsGroupGetStatus: returning 22

2011-04-29 03:10:04.055: [    CRSD][1331140928] CRSD exiting on CSS shutdown request
2011-04-29 03:10:04.055: [    CRSD][1331140928] Done.

2011-04-29 03:40:18.889: [ default][109424896] CRS Daemon Starting
2011-04-29 03:40:18.899: [ CRSMAIN][109424896] Checking the OCR device
2011-04-29 03:40:19.251: [ CRSMAIN][109424896] Connecting to the CSS Daemon
2011-04-29 03:40:19.433: [ COMMCRS][1110460736]clsc_connect: (0x34c23d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-29 03:40:19.433: [ CSSCLNT][109424896]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-29 03:40:19.433: [  CRSRTI][109424896] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-29 03:40:20.613: [ COMMCRS][1110460736]clsc_connect: (0x34c23d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-29 03:40:20.613: [ CSSCLNT][109424896]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-29 03:40:20.620: [  CRSRTI][109424896] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-29 03:40:21.800: [ COMMCRS][1110460736]clsc_connect: (0x34c23d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-29 03:40:21.800: [ CSSCLNT][109424896]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-29 03:40:21.800: [  CRSRTI][109424896] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-29 03:40:22.981: [ COMMCRS][1110460736]clsc_connect: (0x34c23d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-04-29 03:40:22.981: [ CSSCLNT][109424896]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-04-29 03:40:22.981: [  CRSRTI][109424896] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-04-29 03:40:25.153: [ CRSMAIN][109424896] CRSD running as the Privileged user

2011-04-29 03:40:25.389: [  CLSVER][109424896] Active Version from OCR:11.1.0.7.0
2011-04-29 03:40:25.389: [  CLSVER][109424896] Active Version and Software Version are same
2011-04-29 03:40:25.389: [ CRSMAIN][109424896] Initializing OCR
2011-04-29 03:40:25.418: [  OCRRAW][109424896]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-04-29 03:40:25.456: [  OCRSRV][109424896]th_init: Failure in smwait. Waiting to be signalled by Master elector thread.
2011-04-29 03:40:25.526: [    CRSD][109424896] ENV Logging level for Module: allcomp  0
2011-04-29 03:40:25.551: [    CRSD][109424896] ENV Logging level for Module: default  0
2011-04-29 03:40:25.553: [    CRSD][109424896] ENV Logging level for Module: COMMCRS  0
2011-04-29 03:40:25.583: [    CRSD][109424896] ENV Logging level for Module: COMMNS  0
2011-04-29 03:40:25.585: [    CRSD][109424896] ENV Logging level for Module: CRSUI  0
2011-04-29 03:40:25.587: [    CRSD][109424896] ENV Logging level for Module: CRSCOMM  0
2011-04-29 03:40:25.591: [    CRSD][109424896] ENV Logging level for Module: CRSRTI  0
2011-04-29 03:40:25.593: [    CRSD][109424896] ENV Logging level for Module: CRSMAIN  0
2011-04-29 03:40:25.596: [    CRSD][109424896] ENV Logging level for Module: CRSPLACE  0
2011-04-29 03:40:25.598: [    CRSD][109424896] ENV Logging level for Module: CRSAPP  0
2011-04-29 03:40:25.602: [    CRSD][109424896] ENV Logging level for Module: CRSRES  0
2011-04-29 03:40:25.606: [    CRSD][109424896] ENV Logging level for Module: CRSOCR  0
2011-04-29 03:40:25.617: [    CRSD][109424896] ENV Logging level for Module: CRSTIMER  0
2011-04-29 03:40:25.619: [    CRSD][109424896] ENV Logging level for Module: CRSEVT  0
2011-04-29 03:40:25.626: [    CRSD][109424896] ENV Logging level for Module: CRSD  0
2011-04-29 03:40:25.636: [    CRSD][109424896] ENV Logging level for Module: CLUCLS  0
2011-04-29 03:40:25.642: [    CRSD][109424896] ENV Logging level for Module: CLSVER  0
2011-04-29 03:40:25.646: [    CRSD][109424896] ENV Logging level for Module: OCRRAW  0
2011-04-29 03:40:25.650: [    CRSD][109424896] ENV Logging level for Module: OCROSD  0
2011-04-29 03:40:25.660: [    CRSD][109424896] ENV Logging level for Module: OCRCAC  0
2011-04-29 03:40:25.662: [    CRSD][109424896] ENV Logging level for Module: CSSCLNT  0
2011-04-29 03:40:25.665: [    CRSD][109424896] ENV Logging level for Module: OCRAPI  0
2011-04-29 03:40:25.669: [    CRSD][109424896] ENV Logging level for Module: OCRUTL  0
2011-04-29 03:40:25.672: [    CRSD][109424896] ENV Logging level for Module: OCRMSG  0
2011-04-29 03:40:25.674: [    CRSD][109424896] ENV Logging level for Module: OCRCLI  0
2011-04-29 03:40:25.676: [    CRSD][109424896] ENV Logging level for Module: OCRSRV  0
2011-04-29 03:40:25.680: [    CRSD][109424896] ENV Logging level for Module: OCRMAS  0
2011-04-29 03:40:25.681: [ CRSMAIN][109424896] Filename is /u01/crs/11.1.0/db_1/crs/init/rac02.pid
[  clsdmt][1414666560]Listening to (ADDRESS=(PROTOCOL=ipc)(KEY=rac02DBG_CRSD))
2011-04-29 03:40:25.738: [ CRSMAIN][109424896] Using Authorizer location: /u01/crs/11.1.0/db_1/crs/auth/
2011-04-29 03:40:25.813: [ CRSMAIN][109424896] Initializing RTI
2011-04-29 03:40:25.912: [ CRSMAIN][109424896] Initializing EVMMgr
2011-04-29 03:40:25.912: [CRSTIMER][1435646272] Timer Thread Starting.
2011-04-29 03:40:26.607: [ CRSMAIN][109424896] CRSD locked during state recovery, please wait.
2011-04-29 03:40:26.835: [ CRSMAIN][109424896] CRSD recovered, unlocked.
2011-04-29 03:40:26.836: [ CRSMAIN][109424896] QS socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=ora_crsqs))
2011-04-29 03:40:26.865: [ CRSMAIN][109424896] CRSD UI socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=CRSD_UI_SOCKET))
2011-04-29 03:40:26.949: [ CRSMAIN][109424896] E2E socket on: (ADDRESS=(PROTOCOL=tcp)(HOST=rac02-priv)(PORT=49896))
2011-04-29 03:40:26.949: [ CRSMAIN][109424896] Starting Threads
2011-04-29 03:40:26.950: [ CRSMAIN][109424896] CRS Daemon Started.
2011-04-29 03:40:26.950: [ CRSMAIN][1490196800] Starting runCommandServer for (UI = 1, E2E = 0). 0
2011-04-29 03:40:26.950: [ CRSMAIN][1492298048] Starting runCommandServer for (UI = 1, E2E = 0). 1
2011-04-29 03:40:27.025: [  CRSRES][109424896]  startup = 1
2011-04-29 03:40:27.050: [  CRSRES][109424896]  startup = 1
2011-04-29 03:40:27.091: [  CRSRES][109424896]  startup = 1
2011-04-29 03:40:27.116: [  CRSRES][109424896]  startup = 1
2011-04-29 03:40:27.136: [  CRSRES][109424896]  startup = 1
2011-04-29 03:40:27.200: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:27.218: [  CRSRES][1498601792] Attempting to stop `ora.rac02.vip` on member `rac01`
2011-04-29 03:40:27.500: [  CRSRES][1498601792] Stop of `ora.rac02.vip` on member `rac01` succeeded.
2011-04-29 03:40:27.512: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:27.512: [  CRSRES][1498601792] Attempting to start `ora.rac02.vip` on member `rac02`
2011-04-29 03:40:31.139: [  CRSRES][1498601792] Start of `ora.rac02.vip` on member `rac02` succeeded.
2011-04-29 03:40:31.239: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:31.277: [  CRSRES][1498601792] Attempting to start `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-04-29 03:40:35.544: [  CRSRES][1498601792] Start of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-04-29 03:40:35.644: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:35.706: [  CRSRES][1098410304] startRunnable: setting CLI values
2011-04-29 03:40:35.731: [  CRSRES][1498601792] Attempting to stop `pcard.listener` on member `rac01`
2011-04-29 03:40:35.748: [  CRSRES][1532172608] startRunnable: setting CLI values
2011-04-29 03:40:35.761: [  CRSRES][1098410304] Attempting to start `ora.rac02.gsd` on member `rac02`
2011-04-29 03:40:35.797: [  CRSRES][1498601792] Stop of `pcard.listener` on member `rac01` succeeded.
2011-04-29 03:40:35.798: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:35.811: [  CRSRES][1532172608] Attempting to start `ora.rac02.ons` on member `rac02`
2011-04-29 03:40:35.854: [  CRSRES][1498601792] Attempting to stop `pcard.vip` on member `rac01`
2011-04-29 03:40:35.932: [  CRSRES][1498601792] Stop of `pcard.vip` on member `rac01` succeeded.
2011-04-29 03:40:35.933: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:35.997: [  CRSRES][1498601792] Attempting to stop `pcard` on member `rac01`
2011-04-29 03:40:36.033: [  CRSRES][1498601792] Stop of `pcard` on member `rac01` succeeded.
2011-04-29 03:40:36.079: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:36.079: [  CRSRES][1498601792] Attempting to start `pcard` on member `rac02`
2011-04-29 03:40:36.214: [  CRSRES][1498601792] Start of `pcard` on member `rac02` succeeded.
2011-04-29 03:40:36.222: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:36.223: [  CRSRES][1498601792] Attempting to start `pcard.vip` on member `rac02`
2011-04-29 03:40:36.390: [  CRSRES][1534273856] CRS-1002: Resource 'ora.rac02.LISTENER_RAC02.lsnr' is already running on member 'rac02'

2011-04-29 03:40:36.404: [  CRSRES][1098410304] Start of `ora.rac02.gsd` on member `rac02` succeeded.
2011-04-29 03:40:37.557: [  CRSRES][1532172608] Start of `ora.rac02.ons` on member `rac02` succeeded.
2011-04-29 03:40:39.351: [  CRSRES][1498601792] Start of `pcard.vip` on member `rac02` succeeded.
2011-04-29 03:40:39.368: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:39.369: [  CRSRES][1498601792] Attempting to start `pcard.listener` on member `rac02`
2011-04-29 03:40:42.479: [  CRSRES][1498601792] Start of `pcard.listener` on member `rac02` succeeded.
2011-04-29 03:40:42.497: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:42.529: [  CRSRES][1498601792] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-29 03:40:45.118: [  CRSAPP][1498601792] StartResource error for pcard.db_pcard error code = 1
2011-04-29 03:40:45.228: [  CRSRES][1498601792] Start of `pcard.db_pcard` on member `rac02` failed.
2011-04-29 03:40:45.229: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:45.240: [  CRSRES][1498601792] Attempting to stop `pcard.listener` on member `rac02`
2011-04-29 03:40:45.341: [  CRSRES][1498601792] Stop of `pcard.listener` on member `rac02` succeeded.
2011-04-29 03:40:45.342: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:45.351: [  CRSRES][1498601792] Attempting to stop `pcard.vip` on member `rac02`
2011-04-29 03:40:45.416: [  CRSRES][1498601792] Stop of `pcard.vip` on member `rac02` succeeded.
2011-04-29 03:40:45.417: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:45.426: [  CRSRES][1498601792] Attempting to stop `pcard` on member `rac02`
2011-04-29 03:40:45.441: [  CRSRES][1498601792] Stop of `pcard` on member `rac02` succeeded.
2011-04-29 03:40:45.555: [  CRSRES][1498601792] Attempting to start `pcard` on member `rac01`
2011-04-29 03:40:45.574: [  CRSRES][1498601792] Start of `pcard` on member `rac01` succeeded.
2011-04-29 03:40:45.628: [  CRSRES][1498601792] Attempting to start `pcard.vip` on member `rac01`
2011-04-29 03:40:48.715: [  CRSRES][1498601792] Start of `pcard.vip` on member `rac01` succeeded.
2011-04-29 03:40:48.745: [  CRSRES][1498601792] Attempting to start `pcard.listener` on member `rac01`
2011-04-29 03:40:51.809: [  CRSRES][1498601792] Start of `pcard.listener` on member `rac01` succeeded.
2011-04-29 03:40:51.912: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:51.919: [  CRSRES][1498601792] Attempting to stop `pcard.listener` on member `rac01`
2011-04-29 03:40:51.954: [  CRSRES][1498601792] Stop of `pcard.listener` on member `rac01` succeeded.
2011-04-29 03:40:51.955: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:51.958: [  CRSRES][1498601792] Attempting to stop `pcard.vip` on member `rac01`
2011-04-29 03:40:52.025: [  CRSRES][1498601792] Stop of `pcard.vip` on member `rac01` succeeded.
2011-04-29 03:40:52.026: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:52.034: [  CRSRES][1498601792] Attempting to stop `pcard` on member `rac01`
2011-04-29 03:40:52.056: [  CRSRES][1498601792] Stop of `pcard` on member `rac01` succeeded.
2011-04-29 03:40:52.065: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:52.066: [  CRSRES][1498601792] Attempting to start `pcard` on member `rac02`
2011-04-29 03:40:52.086: [  CRSRES][1498601792] Start of `pcard` on member `rac02` succeeded.
2011-04-29 03:40:52.095: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:52.096: [  CRSRES][1498601792] Attempting to start `pcard.vip` on member `rac02`
2011-04-29 03:40:55.170: [  CRSRES][1498601792] Start of `pcard.vip` on member `rac02` succeeded.
2011-04-29 03:40:55.186: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:55.186: [  CRSRES][1498601792] Attempting to start `pcard.listener` on member `rac02`
2011-04-29 03:40:58.280: [  CRSRES][1498601792] Start of `pcard.listener` on member `rac02` succeeded.
2011-04-29 03:40:58.291: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-04-29 03:40:58.297: [  CRSRES][1498601792] Attempting to start `pcard.db_pcard` on member `rac02`
2011-04-29 03:40:58.738: [  CRSAPP][1498601792] StartResource error for pcard.db_pcard error code = 1
2011-04-29 03:40:58.919: [  CRSRES][1498601792] Start of `pcard.db_pcard` on member `rac02` failed.
2011-04-29 03:40:58.919: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:58.920: [  CRSRES][1498601792] Attempting to stop `pcard.listener` on member `rac02`
2011-04-29 03:40:58.962: [  CRSRES][1498601792] Stop of `pcard.listener` on member `rac02` succeeded.
2011-04-29 03:40:58.963: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:58.964: [  CRSRES][1498601792] Attempting to stop `pcard.vip` on member `rac02`
2011-04-29 03:40:59.073: [  CRSRES][1498601792] Stop of `pcard.vip` on member `rac02` succeeded.
2011-04-29 03:40:59.074: [  CRSRES][1498601792] StopResource: setting CLI values
2011-04-29 03:40:59.074: [  CRSRES][1498601792] Attempting to stop `pcard` on member `rac02`
2011-04-29 03:40:59.100: [  CRSRES][1498601792] Stop of `pcard` on member `rac02` succeeded.
2011-04-29 03:40:59.183: [  CRSRES][1498601792] Attempting to start `pcard` on member `rac01`
2011-04-29 03:40:59.242: [  CRSRES][1498601792] Start of `pcard` on member `rac01` succeeded.
2011-04-29 03:40:59.251: [  CRSRES][1498601792] Attempting to start `pcard.vip` on member `rac01`
2011-04-29 03:41:02.326: [  CRSRES][1498601792] Start of `pcard.vip` on member `rac01` succeeded.
2011-04-29 03:41:02.334: [  CRSRES][1498601792] Attempting to start `pcard.listener` on member `rac01`
2011-04-29 03:41:05.404: [  CRSRES][1498601792] Start of `pcard.listener` on member `rac01` succeeded.
2011-04-29 04:08:43.007: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:08:43.022: [  CRSRES][1098410304] startRunnable: setting CLI values
2011-04-29 04:08:43.047: [    CRSD][1098410304] SM:dE2Ec: all E2E cmds done. 0
2011-04-29 04:08:43.049: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:08:43.070: [  CRSRES][1098410304] startRunnable: setting CLI values
2011-04-29 04:08:46.169: [    CRSD][1098410304] SM:dE2Ec: all E2E cmds done. 0
2011-04-29 04:08:46.172: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:08:46.187: [  CRSRES][1098410304] startRunnable: setting CLI values
2011-04-29 04:08:49.257: [    CRSD][1098410304] SM:dE2Ec: all E2E cmds done. 0
2011-04-29 04:08:49.259: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:08:49.272: [  CRSRES][1098410304] startRunnable: setting CLI values
2011-04-29 04:09:23.309: [    CRSD][1098410304] SM:dE2Ec: all E2E cmds done. 0
2011-04-29 04:09:23.312: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:09:23.332: [  CRSRES][1098410304] startRunnable: setting CLI values
2011-04-29 04:09:23.500: [    CRSD][1098410304] SM:dE2Ec: all E2E cmds done. 0
2011-04-29 04:10:20.726: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:10:20.726: [  CRSRES][1532172608] StopResource: setting CLI values
2011-04-29 04:10:20.786: [    CRSD][1532172608] SM:dE2Ec: all E2E cmds done. 0
2011-04-29 04:10:20.804: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:10:20.804: [  CRSRES][1532172608] StopResource: setting CLI values
2011-04-29 04:10:49.099: [    CRSD][1532172608] SM:dE2Ec: all E2E cmds done. 0
2011-04-29 04:10:49.116: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:10:49.116: [  CRSRES][1532172608] StopResource: setting CLI values
2011-04-29 04:10:49.161: [    CRSD][1532172608] SM:dE2Ec: all E2E cmds done. 0
2011-04-29 04:10:49.176: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:10:49.176: [  CRSRES][1532172608] StopResource: setting CLI values
2011-04-29 04:10:49.247: [    CRSD][1532172608] SM:dE2Ec: all E2E cmds done. 0
2011-04-29 04:10:49.249: [    CRSD][1490196800] SM: rE2Ec: 4
2011-04-29 04:10:49.249: [  CRSRES][1532172608] StopResource: setting CLI values
2011-04-29 04:10:49.269: [    CRSD][1532172608] SM:dE2Ec: all E2E cmds done. 0
2011-05-02 21:03:35.101: [  OCRSRV][1372707136]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.101: [  OCRSRV][1288788288]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.101: [  OCRSRV][1341237568]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.101: [  OCRSRV][1299278144]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.102: [  OCRSRV][1257318720]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.101: [  OCRSRV][1330747712]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.101: [  OCRSRV][1267808576]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.101: [  OCRSRV][1351727424]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.102: [  OCRSRV][1320257856]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.102: [  OCRRAW][1120950592]pr_io_wait: Error in smwait. retry.
2011-05-02 21:03:35.101: [  OCRSRV][1362217280]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.102: [  OCRSRV][1278298432]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.102: [  OCRSRV][1309768000]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.101: [  OCRSRV][1383196992]th_select_w_f_r: smwait error [1]
2011-05-02 21:03:35.101: [  OCRSRV][1393686848]th_select_w_f_r: smwait error [1]
2011-05-03 17:22:24.672: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:22:24.688: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-03 17:22:24.707: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-03 17:22:24.709: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:22:24.717: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-03 17:22:27.798: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-03 17:22:27.800: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:22:27.809: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-03 17:22:30.884: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-03 17:22:30.886: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:22:30.895: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-03 17:22:47.675: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-03 17:22:47.678: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:22:47.689: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-03 17:22:47.707: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-03 17:23:08.111: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:23:08.112: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-03 17:23:08.133: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-03 17:23:08.141: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:23:08.142: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-03 17:23:32.235: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-03 17:23:32.252: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:23:32.252: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-03 17:23:32.300: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-03 17:23:32.317: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:23:32.317: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-03 17:23:32.390: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-03 17:23:32.392: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-03 17:23:32.392: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-03 17:23:32.413: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:31:59.346: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:31:59.358: [  CRSRES][1498601792] Attempting to stop `pcard.listener` on member `rac01`
2011-05-04 20:31:59.399: [  CRSRES][1498601792] Stop of `pcard.listener` on member `rac01` succeeded.
2011-05-04 20:31:59.400: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:31:59.416: [  CRSRES][1498601792] Attempting to stop `pcard.vip` on member `rac01`
2011-05-04 20:31:59.486: [  CRSRES][1498601792] Stop of `pcard.vip` on member `rac01` succeeded.
2011-05-04 20:31:59.487: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:31:59.500: [  CRSRES][1498601792] Attempting to stop `pcard` on member `rac01`
2011-05-04 20:31:59.521: [  CRSRES][1498601792] Stop of `pcard` on member `rac01` succeeded.
2011-05-04 20:31:59.533: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-04 20:31:59.534: [  CRSRES][1498601792] Attempting to start `pcard` on member `rac02`
2011-05-04 20:31:59.553: [  CRSRES][1498601792] Start of `pcard` on member `rac02` succeeded.
2011-05-04 20:31:59.565: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-04 20:31:59.566: [  CRSRES][1498601792] Attempting to start `pcard.vip` on member `rac02`
2011-05-04 20:32:02.647: [  CRSRES][1498601792] Start of `pcard.vip` on member `rac02` succeeded.
2011-05-04 20:32:02.656: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-04 20:32:02.657: [  CRSRES][1498601792] Attempting to start `pcard.listener` on member `rac02`
2011-05-04 20:32:05.719: [  CRSRES][1498601792] Start of `pcard.listener` on member `rac02` succeeded.
2011-05-04 20:32:14.315: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 20:32:14.316: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:32:14.357: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:32:14.359: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 20:32:14.360: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:32:14.432: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:32:14.434: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 20:32:14.435: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:32:14.463: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:33:22.758: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:33:22.778: [  CRSRES][1498601792] Attempting to stop `pcard.listener` on member `rac01`
2011-05-04 20:33:22.822: [  CRSRES][1498601792] Stop of `pcard.listener` on member `rac01` succeeded.
2011-05-04 20:33:22.823: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:33:22.835: [  CRSRES][1498601792] Attempting to stop `pcard.vip` on member `rac01`
2011-05-04 20:33:22.942: [  CRSRES][1498601792] Stop of `pcard.vip` on member `rac01` succeeded.
2011-05-04 20:33:22.943: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:33:22.946: [  CRSRES][1498601792] Attempting to stop `pcard` on member `rac01`
2011-05-04 20:33:22.973: [  CRSRES][1498601792] Stop of `pcard` on member `rac01` succeeded.
2011-05-04 20:33:22.981: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-04 20:33:22.982: [  CRSRES][1498601792] Attempting to start `pcard` on member `rac02`
2011-05-04 20:33:23.002: [  CRSRES][1498601792] Start of `pcard` on member `rac02` succeeded.
2011-05-04 20:33:23.016: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-04 20:33:23.017: [  CRSRES][1498601792] Attempting to start `pcard.vip` on member `rac02`
2011-05-04 20:33:26.088: [  CRSRES][1498601792] Start of `pcard.vip` on member `rac02` succeeded.
2011-05-04 20:33:26.097: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-04 20:33:26.098: [  CRSRES][1498601792] Attempting to start `pcard.listener` on member `rac02`
2011-05-04 20:33:29.160: [  CRSRES][1498601792] Start of `pcard.listener` on member `rac02` succeeded.
2011-05-04 20:33:48.230: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 20:33:48.231: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:33:48.276: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:33:48.278: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 20:33:48.279: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:33:48.349: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:33:48.351: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 20:33:48.352: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 20:33:48.375: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:57:50.422: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 20:57:50.435: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-04 20:57:50.459: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:57:50.461: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 20:57:50.501: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-04 20:57:53.579: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:57:53.582: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 20:57:53.593: [  CRSRES][1498601792] startRunnable: setting CLI values
2011-05-04 20:57:56.658: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 20:59:57.484: [  OCRMAS][1225849152]th_master:13: I AM THE NEW OCR MASTER at incar 2. Node Number 2
2011-05-04 20:59:57.554: [  OCRRAW][1225849152]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-05-04 20:59:57.614: [  OCRMAS][1225849152]th_master: Deleted ver keys from cache (master)
2011-05-04 21:00:07.132: [ CRSCOMM][1498601792] CLEANUP: Searching for connections to failed node rac01
2011-05-04 21:00:07.132: [  CRSEVT][1498601792] Processing member leave for rac01, incarnation: 196435638
2011-05-04 21:00:07.133: [    CRSD][1498601792] SM: recovery in process: 8
2011-05-04 21:00:07.133: [  CRSEVT][1498601792] Do failover for: rac01
2011-05-04 21:00:07.379: [  CRSRES][1498601792]  startup = 0
2011-05-04 21:00:07.404: [  CRSRES][1498601792]  startup = 0
2011-05-04 21:00:07.418: [  CRSRES][1498601792]  startup = 0
2011-05-04 21:00:07.479: [  CRSRES][1498601792]  startup = 0
2011-05-04 21:00:07.496: [  CRSRES][1498601792]  startup = 0
2011-05-04 21:00:07.524: [  CRSRES][1498601792]  startup = 0
2011-05-04 21:00:07.783: [  CRSRES][1532172608] startRunnable: setting CLI values
2011-05-04 21:00:07.832: [  CRSRES][1532172608] Attempting to start `pcard.db_pcard` on member `rac02`
2011-05-04 21:00:07.943: [  CRSRES][1098410304] startRunnable: setting CLI values
2011-05-04 21:00:08.045: [  CRSRES][1098410304] Attempting to start `ora.rac01.vip` on member `rac02`
2011-05-04 21:00:11.579: [  CRSRES][1098410304] Start of `ora.rac01.vip` on member `rac02` succeeded.
2011-05-04 21:00:50.292: [  CRSRES][1532172608] Start of `pcard.db_pcard` on member `rac02` succeeded.
2011-05-04 21:00:50.341: [  CRSRES][1532172608] startRunnable: setting CLI values
2011-05-04 21:00:50.348: [  CRSRES][1532172608] Attempting to start `pcard.head` on member `rac02`
2011-05-04 21:00:50.383: [  CRSRES][1532172608] Start of `pcard.head` on member `rac02` succeeded.
2011-05-04 21:00:50.385: [  CRSEVT][1498601792] Post recovery done evmd event for: rac01
2011-05-04 21:00:50.385: [    CRSD][1498601792] SM: recoveryDone: 0
2011-05-04 21:00:50.387: [  CRSEVT][1498601792] Processing RecoveryDone
2011-05-04 21:00:50.635: [    CRSD][1490196800] SM: rE2Ec: 4
2011-05-04 21:00:50.635: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 21:00:50.892: [    CRSD][1498601792] SM:dE2Ec: all E2E cmds done. 0
2011-05-04 21:01:45.716: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 21:01:45.722: [  CRSRES][1498601792] Attempting to stop `pcard.head` on member `rac02`
2011-05-04 21:01:45.741: [  CRSRES][1498601792] Stop of `pcard.head` on member `rac02` succeeded.
2011-05-04 21:01:45.742: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 21:01:45.749: [  CRSRES][1498601792] Attempting to stop `pcard.db_pcard` on member `rac02`
2011-05-04 21:02:18.686: [  CRSRES][1498601792] Stop of `pcard.db_pcard` on member `rac02` succeeded.
2011-05-04 21:02:18.687: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 21:02:18.712: [  CRSRES][1498601792] Attempting to stop `pcard.listener` on member `rac02`
2011-05-04 21:02:18.760: [  CRSRES][1498601792] Stop of `pcard.listener` on member `rac02` succeeded.
2011-05-04 21:02:18.761: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 21:02:18.770: [  CRSRES][1498601792] Attempting to stop `pcard.vip` on member `rac02`
2011-05-04 21:02:18.839: [  CRSRES][1498601792] Stop of `pcard.vip` on member `rac02` succeeded.
2011-05-04 21:02:18.840: [  CRSRES][1498601792] StopResource: setting CLI values
2011-05-04 21:02:18.861: [  CRSRES][1498601792] Attempting to stop `pcard` on member `rac02`
2011-05-04 21:02:18.882: [  CRSRES][1498601792] Stop of `pcard` on member `rac02` succeeded.
2011-05-04 21:02:18.935: [  CRSRES][1498601792] Attempting to start `pcard` on member `rac01`
2011-05-04 21:02:19.033: [  CRSRES][1498601792] Start of `pcard` on member `rac01` succeeded.
2011-05-04 21:02:19.054: [  CRSRES][1498601792] Attempting to start `pcard.vip` on member `rac01`
2011-05-04 21:02:24.240: [  CRSRES][1498601792] Start of `pcard.vip` on member `rac01` succeeded.
2011-05-04 21:02:24.262: [  CRSRES][1498601792] Attempting to start `pcard.listener` on member `rac01`
2011-05-04 21:02:27.411: [  CRSRES][1498601792] Start of `pcard.listener` on member `rac01` succeeded.
2011-05-04 21:02:27.435: [  CRSRES][1498601792] Attempting to start `pcard.db_pcard` on member `rac01`
2011-05-04 21:02:50.310: [  CRSRES][1498601792] Start of `pcard.db_pcard` on member `rac01` succeeded.
2011-05-04 21:02:50.336: [  CRSRES][1498601792] Attempting to start `pcard.head` on member `rac01`
2011-05-04 21:02:50.460: [  CRSRES][1498601792] Start of `pcard.head` on member `rac01` succeeded.
2011-05-04 23:35:13.121: [ default][2306085120] CRS Daemon Starting
2011-05-04 23:35:13.130: [ CRSMAIN][2306085120] Checking the OCR device
2011-05-04 23:35:13.208: [ CRSMAIN][2306085120] Connecting to the CSS Daemon
2011-05-04 23:35:13.389: [ COMMCRS][1114216768]clsc_connect: (0x2d673d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-05-04 23:35:13.389: [ CSSCLNT][2306085120]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-05-04 23:35:13.390: [  CRSRTI][2306085120] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-05-04 23:35:14.570: [ COMMCRS][1114216768]clsc_connect: (0x2d673d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-05-04 23:35:14.570: [ CSSCLNT][2306085120]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-05-04 23:35:14.570: [  CRSRTI][2306085120] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-05-04 23:35:15.751: [ COMMCRS][1114216768]clsc_connect: (0x2d673d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-05-04 23:35:15.751: [ CSSCLNT][2306085120]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-05-04 23:35:15.751: [  CRSRTI][2306085120] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-05-04 23:35:16.936: [ COMMCRS][1114216768]clsc_connect: (0x2d673d0) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_))

2011-05-04 23:35:16.936: [ CSSCLNT][2306085120]clsssInitNative: failed to connect to (ADDRESS=(PROTOCOL=ipc)(KEY=OCSSD_LL_rac02_)), rc 9

2011-05-04 23:35:16.937: [  CRSRTI][2306085120] CSS is not ready. Received status 3 from CSS. Waiting for good status .. 

2011-05-04 23:35:18.747: [ CRSMAIN][2306085120] CRSD running as the Privileged user

2011-05-04 23:35:18.987: [  CLSVER][2306085120] Active Version from OCR:11.1.0.7.0
2011-05-04 23:35:18.987: [  CLSVER][2306085120] Active Version and Software Version are same
2011-05-04 23:35:18.987: [ CRSMAIN][2306085120] Initializing OCR
2011-05-04 23:35:19.018: [  OCRRAW][2306085120]proprioo: for disk 0 (/dev/mpath/mpath0p1), id match (1), my id set (364465018,1028247821) total id sets (1), 1st set (364465018,1028247821), 2nd set (0,0) my votes (2), total votes (2)
2011-05-04 23:35:19.065: [  OCRSRV][2306085120]th_init: Failure in smwait. Waiting to be signalled by Master elector thread.
2011-05-04 23:35:19.099: [    CRSD][2306085120] ENV Logging level for Module: allcomp  0
2011-05-04 23:35:19.165: [    CRSD][2306085120] ENV Logging level for Module: default  0
2011-05-04 23:35:19.182: [    CRSD][2306085120] ENV Logging level for Module: COMMCRS  0
2011-05-04 23:35:19.184: [    CRSD][2306085120] ENV Logging level for Module: COMMNS  0
2011-05-04 23:35:19.189: [    CRSD][2306085120] ENV Logging level for Module: CRSUI  0
2011-05-04 23:35:19.190: [    CRSD][2306085120] ENV Logging level for Module: CRSCOMM  0
2011-05-04 23:35:19.192: [    CRSD][2306085120] ENV Logging level for Module: CRSRTI  0
2011-05-04 23:35:19.196: [    CRSD][2306085120] ENV Logging level for Module: CRSMAIN  0
2011-05-04 23:35:19.198: [    CRSD][2306085120] ENV Logging level for Module: CRSPLACE  0
2011-05-04 23:35:19.201: [    CRSD][2306085120] ENV Logging level for Module: CRSAPP  0
2011-05-04 23:35:19.203: [    CRSD][2306085120] ENV Logging level for Module: CRSRES  0
2011-05-04 23:35:19.206: [    CRSD][2306085120] ENV Logging level for Module: CRSOCR  0
2011-05-04 23:35:19.209: [    CRSD][2306085120] ENV Logging level for Module: CRSTIMER  0
2011-05-04 23:35:19.211: [    CRSD][2306085120] ENV Logging level for Module: CRSEVT  0
2011-05-04 23:35:19.216: [    CRSD][2306085120] ENV Logging level for Module: CRSD  0
2011-05-04 23:35:19.218: [    CRSD][2306085120] ENV Logging level for Module: CLUCLS  0
2011-05-04 23:35:19.220: [    CRSD][2306085120] ENV Logging level for Module: CLSVER  0
2011-05-04 23:35:19.222: [    CRSD][2306085120] ENV Logging level for Module: OCRRAW  0
2011-05-04 23:35:19.225: [    CRSD][2306085120] ENV Logging level for Module: OCROSD  0
2011-05-04 23:35:19.227: [    CRSD][2306085120] ENV Logging level for Module: OCRCAC  0
2011-05-04 23:35:19.228: [    CRSD][2306085120] ENV Logging level for Module: CSSCLNT  0
2011-05-04 23:35:19.230: [    CRSD][2306085120] ENV Logging level for Module: OCRAPI  0
2011-05-04 23:35:19.231: [    CRSD][2306085120] ENV Logging level for Module: OCRUTL  0
2011-05-04 23:35:19.238: [    CRSD][2306085120] ENV Logging level for Module: OCRMSG  0
2011-05-04 23:35:19.240: [    CRSD][2306085120] ENV Logging level for Module: OCRCLI  0
2011-05-04 23:35:19.241: [    CRSD][2306085120] ENV Logging level for Module: OCRSRV  0
2011-05-04 23:35:19.243: [    CRSD][2306085120] ENV Logging level for Module: OCRMAS  0
2011-05-04 23:35:19.243: [ CRSMAIN][2306085120] Filename is /u01/crs/11.1.0/db_1/crs/init/rac02.pid
[  clsdmt][1418422592]Listening to (ADDRESS=(PROTOCOL=ipc)(KEY=rac02DBG_CRSD))
2011-05-04 23:35:19.287: [ CRSMAIN][2306085120] Using Authorizer location: /u01/crs/11.1.0/db_1/crs/auth/
2011-05-04 23:35:19.357: [ CRSMAIN][2306085120] Initializing RTI
2011-05-04 23:35:19.468: [ CRSMAIN][2306085120] Initializing EVMMgr
2011-05-04 23:35:19.468: [CRSTIMER][1439402304] Timer Thread Starting.
2011-05-04 23:35:19.649: [ COMMCRS][1449892160]clsc_connect: (0x2aaaac01ad90) no listener at (ADDRESS=(PROTOCOL=ipc)(KEY=SYSTEM.evm.acceptor.auth))

2011-05-04 23:35:20.973: [ CRSMAIN][2306085120] CRSD locked during state recovery, please wait.
2011-05-04 23:35:21.115: [ CRSMAIN][2306085120] CRSD recovered, unlocked.
2011-05-04 23:35:21.116: [ CRSMAIN][2306085120] QS socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=ora_crsqs))
2011-05-04 23:35:21.126: [ CRSMAIN][2306085120] CRSD UI socket on: (ADDRESS=(PROTOCOL=ipc)(KEY=CRSD_UI_SOCKET))
2011-05-04 23:35:21.221: [ CRSMAIN][2306085120] E2E socket on: (ADDRESS=(PROTOCOL=tcp)(HOST=rac02-priv)(PORT=49896))
2011-05-04 23:35:21.221: [ CRSMAIN][2306085120] Starting Threads
2011-05-04 23:35:21.221: [ CRSMAIN][2306085120] CRS Daemon Started.
2011-05-04 23:35:21.221: [ CRSMAIN][1084381504] Starting runCommandServer for (UI = 1, E2E = 0). 0
2011-05-04 23:35:21.221: [ CRSMAIN][1102948672] Starting runCommandServer for (UI = 1, E2E = 0). 1
2011-05-04 23:35:21.285: [  CRSRES][2306085120]  startup = 1
2011-05-04 23:35:21.318: [  CRSRES][2306085120]  startup = 1
2011-05-04 23:35:21.686: [  CRSRES][2306085120]  startup = 1
2011-05-04 23:35:21.777: [  CRSRES][1498155328] StopResource: setting CLI values
2011-05-04 23:35:21.800: [  CRSRES][1498155328] Attempting to stop `ora.rac02.vip` on member `rac01`
2011-05-04 23:35:22.082: [  CRSRES][1498155328] Stop of `ora.rac02.vip` on member `rac01` succeeded.
2011-05-04 23:35:22.097: [  CRSRES][1498155328] startRunnable: setting CLI values
2011-05-04 23:35:22.097: [  CRSRES][1498155328] Attempting to start `ora.rac02.vip` on member `rac02`
2011-05-04 23:35:25.590: [  CRSRES][1498155328] Start of `ora.rac02.vip` on member `rac02` succeeded.
2011-05-04 23:35:25.663: [  CRSRES][1498155328] startRunnable: setting CLI values
2011-05-04 23:35:25.688: [  CRSRES][1498155328] Attempting to start `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02`
2011-05-04 23:35:29.960: [  CRSRES][1498155328] Start of `ora.rac02.LISTENER_RAC02.lsnr` on member `rac02` succeeded.
2011-05-04 23:35:30.032: [  CRSRES][1498155328] startRunnable: setting CLI values
2011-05-04 23:35:30.073: [  CRSRES][1498155328] Attempting to start `ora.rac02.gsd` on member `rac02`
2011-05-04 23:35:30.080: [  CRSRES][1080392000] startRunnable: setting CLI values
2011-05-04 23:35:30.109: [  CRSRES][1080392000] Attempting to start `ora.rac02.ons` on member `rac02`
2011-05-04 23:35:30.680: [  CRSRES][1498155328] Start of `ora.rac02.gsd` on member `rac02` succeeded.
2011-05-04 23:35:30.873: [  CRSRES][1531726144] CRS-1002: Resource 'ora.rac02.LISTENER_RAC02.lsnr' is already running on member 'rac02'

2011-05-04 23:35:31.832: [  CRSRES][1080392000] Start of `ora.rac02.ons` on member `rac02` succeeded.
</verbatim>
[[#Top][Back to top]]
---+ Last 2000 Lines of crsdOUT.log File
---## Information Taken from /u01/crs/11.1.0/db_1/log/rac02/crsd/crsdOUT.log
<verbatim>
2011-03-24 16:13:35   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-03-24 16:13:35   : CRSD REBOOT
2011-03-24 16:48:34   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-03-24 16:48:34   : CRSD REBOOT
2011-03-24 16:49:41   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-03-24 16:49:41   : CRSD REBOOT
2011-03-24 17:17:10   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-03-24 17:17:10   : CRSD REBOOT
2011-03-26 18:43:57   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-03-26 18:43:57   : CRSD REBOOT
2011-03-28 00:52:30   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-03-28 00:52:30   : CRSD REBOOT
2011-04-03 13:40:21   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-03 13:40:21   : CRSD REBOOT
2011-04-05 05:33:20   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-05 05:33:20   : CRSD REBOOT
2011-04-09 02:26:35   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-09 02:26:35   : CRSD REBOOT
2011-04-10 19:48:34   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-10 19:48:34   : CRSD REBOOT
2011-04-14 10:43:10   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-14 10:43:10   : CRSD REBOOT
2011-04-14 12:38:41   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-14 12:38:41   : CRSD REBOOT
2011-04-14 16:08:26   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-14 16:08:26   : CRSD REBOOT
2011-04-19 12:47:58   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-19 12:47:58   : CRSD REBOOT
2011-04-19 12:56:40   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-19 12:56:40   : CRSD REBOOT
2011-04-19 13:23:06   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-19 13:23:06   : CRSD REBOOT
2011-04-26 15:16:42   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-26 15:16:42   : CRSD REBOOT
2011-04-27 05:12:30   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-27 05:12:30   : CRSD REBOOT
2011-04-27 09:09:36   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-27 09:09:36   : CRSD REBOOT
2011-04-27 11:21:48   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-27 11:21:48   : CRSD REBOOT
2011-04-27 11:29:16   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-27 11:29:16   : CRSD REBOOT
2011-04-29 03:40:18   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-04-29 03:40:18   : CRSD REBOOT
2011-05-04 23:35:13   : Changing directory to /u01/crs/11.1.0/db_1/log/rac02/crsd
2011-05-04 23:35:13   : CRSD REBOOT
</verbatim>
[[#Top][Back to top]]
