資料庫執行個體正，資料磁盤組online,asm磁盤組正常通路。
故障現象：
[root@p750db2://]#su - grid
$ crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
$ ps -ef|grep has
root 6029498 1 0 Jan 01 - 1971:22 /oracle/app/11.2.0/grid/bin/ohasd.bin reboot
root 6225994 1 0 Jan 01 - 0:00 /bin/sh /etc/init.ohasd run
grid 23986302 21364882 0 10:49:58 pts/2 0:00 grep has
$ ps -ef|grep pmon
grid 24838256 21364882 0 10:50:07 pts/2 0:00 grep pmon
oracle 14942614 1 0 Aug 31 - 32:24 ora_pmon_db2
grid 21758402 1 0 Aug 31 - 20:40 asm_pmon_+ASM2
$ crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
$ which crsctl
/oracle/app/11.2.0/grid/bin/crsctl
$ exit
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl start crs
CRS-4640: Oracle High Availability Services is already active
CRS-4000: Command Start failed, or completed with errors.
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl check has
CRS-4638: Oracle High Availability Services is online
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE p750db2 Started
ora.cluster_interconnect.haip
1 ONLINE ONLINE p750db2
ora.crf
1 ONLINE ONLINE p750db2
ora.crsd
1 ONLINE OFFLINE
ora.cssd
1 ONLINE ONLINE p750db2
ora.cssdmonitor
1 ONLINE ONLINE p750db2
ora.ctssd
1 ONLINE ONLINE p750db2 OBSERVER
ora.diskmon
1 OFFLINE OFFLINE
ora.drivers.acfs
1 ONLINE ONLINE p750db2
ora.evmd
1 ONLINE ONLINE p750db2
ora.gipcd
1 ONLINE ONLINE p750db2
ora.gpnpd
1 ONLINE ONLINE p750db2
ora.mdnsd
1 ONLINE ONLINE p750db2 故障解決
檢查asm磁盤，發現存放ocr 磁盤組offline
手動mount OCR磁盤組
[root@p750db2://]#su - grid
$ asmcmd
ASMCMD> ls
ARCH/
DATA/
ASMCMD> mount -a
ORA-15032: not all alterations performed
ORA-15017: diskgroup "DATA" cannot be mounted
ORA-15013: diskgroup "DATA" is already mounted
ORA-15017: diskgroup "ARCH" cannot be mounted
ORA-15013: diskgroup "ARCH" is already mounted (DBD ERROR: OCIStmtExecute)
ASMCMD> lsdg
State Type Rebal Sector Block AU Total_MB Free_MB Req_mir_free_MB Usable_file_MB Offline_disks Voting_files Name
MOUNTED EXTERN N 512 4096 1048576 1024000 377578 0 377578 0 N ARCH/
MOUNTED EXTERN N 512 4096 1048576 1024000 516385 0 516385 0 N DATA/
MOUNTED HIGH N 512 4096 1048576 25600 24092 10240 4617 0 Y GRID/
ASMCMD> exit 啟動crsd 程序
$ exit
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl start resource ora.crsd -init
CRS-2672: Attempting to start 'ora.crsd' on 'p750db2'
CRS-2676: Start of 'ora.crsd' on 'p750db2' succeeded 故障解決，檢查叢集狀态
[root@p750db2://]#
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl stat res -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ARCH.dg
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.DATA.dg
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.GRID.dg
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.LISTENER.lsnr
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE OFFLINE p750db4
ora.asm
ONLINE ONLINE p750db1 Started
ONLINE ONLINE p750db2 Started
ONLINE ONLINE p750db3 Started
ONLINE ONLINE p750db4 Started
ora.gsd
OFFLINE OFFLINE p750db1
OFFLINE OFFLINE p750db2
OFFLINE OFFLINE p750db3
OFFLINE OFFLINE p750db4
ora.net1.network
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.ons
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.registry.acfs
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE p750db2
ora.cvu
1 ONLINE ONLINE p750db1
ora.db.db
1 ONLINE ONLINE p750db1 Open
2 ONLINE ONLINE p750db2 Open
3 ONLINE ONLINE p750db3 Open
4 ONLINE ONLINE p750db4 Open
ora.oc4j
1 ONLINE ONLINE p750db1
ora.p750db1.vip
1 ONLINE ONLINE p750db1
ora.p750db2.vip
1 ONLINE ONLINE p750db2
ora.p750db3.vip
1 ONLINE ONLINE p750db3
ora.p750db4.vip
1 ONLINE INTERMEDIATE p750db2 FAILED OVER
ora.scan1.vip
1 ONLINE ONLINE p750db2
[root@p750db2://]# ![Shell程式設計——sort排序、uniq忽略重複、tr替換壓縮删除、cut指定删除字段、正規表達式元字元sort 指令uniq 指令tr 指令cut 指令正規表達式[圖]](data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=)