Login test environment discovers a node listener exception
root@xxxxdbb:/#crsctl Status Res-t
--------------------------------------------------------------------------------
NAME TARGET State SERVER State_details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
Ora.xxxx_DATA.dg
Online online XXXXDBA
Online online XXXXDBB
Ora. Listener.lsnr
Online online XXXXDBA
ONLINE OFFLINE XXXXDBB
Ora.asm
Online online XXXXDBA started
Online online XXXXDBB started
Ora.gsd
OFFLINE OFFLINE XXXXDBA
OFFLINE OFFLINE XXXXDBB
Ora.net1.network
Online online XXXXDBA
Online online XXXXDBB
Ora.ons
Online online XXXXDBA
Online online XXXXDBB
Ora.registry.acfs
Online online XXXXDBA
Online online XXXXDBB
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
Ora. Listener_scan1.lsnr
1 Online Online Xxxxdba
Ora.cvu
1 Online Online Xxxxdba
Ora.xxxxdb.db
1 Online Online xxxxdba Open
2 Online Online XXXXDBB Open
Ora.xxxxdb.xxxx.svc
1 Online Online Xxxxdba
Ora.xxxxdb.sjcj.svc
1 Online Online Xxxxdba
Ora.xxxxdb.zfbf.svc
1 Online Online Xxxxdba
Ora.xxxxdba.vip
1 Online Online Xxxxdba
Ora.xxxxdbb.vip
1 Online Online XXXXDBB
Ora.oc4j
1 Online Online Xxxxdba
Ora.scan1.vip
1 Online Online Xxxxdba
root@xxxxdbb:/#su-grid
Attempt to manually start, same error
Grid@xxxxdbb:/home/grid $lsnrctl Statuss
Lsnrctl for Ibm/aix RISC system/6000:version 11.2.0.4.0-production on 04-mar-2016 01:23:34
Copyright (c) 1991, 2013, Oracle. All rights reserved.
Connecting to (description= address= (PROTOCOL=IPC) (Key=listener))
Tns-12541:tns:no Listener
Tns-12560:tns:protocol Adapter Error
Tns-00511:no Listener
Ibm/aix RISC system/6000 Error:79:connection refused
Grid@xxxxdbb:/home/grid $
Grid@xxxxdbb:/home/grid $cat/etc/hosts
The first suspicion is that the hosts file was changed, see host did not find modified. Direct Truss
From the above it is easy to start using the IPC method, try truss to see if there is information
Truss Lsnrctl Start
Execve ("/usr/bin/lsnrctl", 0x2ff22bf8, 0x200138a8) err#2 enoent
Execve ("/etc/lsnrctl", 0x2ff22bf8, 0x200138a8) err#2 enoent
Execve ("/usr/sbin/lsnrctl", 0x2ff22bf8, 0x200138a8) err#2 enoent
Execve ("/usr/ucb/lsnrctl", 0x2ff22bf8, 0x200138a8) err#2 enoent
Execve ("/home/grid/bin/lsnrctl", 0x2ff22bf8, 0x200138a8) err#2 enoent
Execve ("/usr/bin/x11/lsnrctl", 0x2ff22bf8, 0x200138a8) err#2 enoent
Execve ("/sbin/lsnrctl", 0x2ff22bf8, 0x200138a8) err#2 enoent
Execve ("./lsnrctl", 0x2ff22bf8, 0x200138a8) err#2 enoent
Execve ("/u01/app/11.2.0.4/grid/bin/lsnrctl", 0x2ff22bf8, 0x200138a8) argc:2
Kusla (2, 0x09fffffff0001170) err#1 eperm
Read_sysconfig (0x09001000a07d1550, 0x0000000000000010, 0x0000000000000000, 0x00000000b0000010, 0x0000000110000E38, 0xbadc0ffee0ddf00d, 0x0000000006010000, 0x09001000a0806e68) = 0x0000000000000000
SBRK (0x0000000000000000) = 0x00000001101d54d8
Vmgetinfo (0x0ffffffffffff140, 7, 16) = 0
SBRK (0x0000000000000000) = 0x00000001101d54d8
SBRK (0x0000000000000008) = 0x00000001101d54d8
__LIBC_SBRK (0x0000000000010020) = 0x00000001101d54e0
Thread_init (0x0900000000520760, 0x09001000a0888470) =
SBRK (0x0000000000000000) = 0x00000001101e5500
Vmgetinfo (0x0ffffffffffff780, 7, 16) = 0
Smcr_procattr (0, 1, 0x0ffffffffffff778) err#109 Enosys
Getrpid (-1,-1, 648535941220675944) = 8912948
_getpid () = 8912948
Getprocs64 (0x00000001101d7bf0, 5024, 0x0000000000000000, 0, 0x09001000a0883098, 1) = 1
Appulimit (1005, 0) = 0x0ffffffffe000000
_thread_self () = 66453675
Thread_setmystate (0x0000000000000000, 0x0ffffffffffff2a0) = 0
Thread_setmystate (0x0fffffffffffef00, 0x0ffffffffffff288) = 0
_sigaction (3, 0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
_sigaction (4, 0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
_sigaction (5, 0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
_sigaction (6, 0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
_sigaction (7, 0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
_sigaction (8, 0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
_sigaction (0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
_sigaction (one, 0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
_sigaction (0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
_sigaction (0x0ffffffffffff670, 0x0ffffffffffff6a0) = 0
..... Omitted....
_getpid () = 8912948
ACCESSX ("/etc/secvars.cfg", 04, 0) = 0
Statx ("/etc/secvars.cfg", 0x0fffffffffff6fb0, 176, 0) = 0
_getpid () = 8912948
ACCESSX ("/etc/passwd", 04, 0) = 0
Statx ("/etc/passwd", 0x0fffffffffff6e20, 176, 0) = 0
ACCESSX ("/etc/security/passwd", 0) err#13 eacces
ACCESSX ("/etc/passwd", 04, 0) = 0
Statx ("/etc/passwd", 0x0fffffffffff6b50, 176, 0) = 0
ACCESSX ("/etc/passwd", 04, 0) = 0
Statx ("/etc/passwd", 0x0fffffffffff6c20, 176, 0) = 0
_getpid () = 8912948
ACCESSX ("/etc/passwd", 04, 0) = 0
Statx ("/etc/passwd", 0x0fffffffffff6e20, 176, 0) = 0
ACCESSX ("/etc/security/passwd", 0) err#13 eacces
ACCESSX ("/etc/passwd", 04, 0) = 0
Statx ("/etc/passwd", 0x0fffffffffff6b50, 176, 0) = 0
ACCESSX ("/etc/passwd", 04, 0) = 0
Statx ("/etc/passwd", 0x0fffffffffff6c20, 176, 0) = 0
ACCESSX ("/etc/security/passwd", 0) err#13 eacces
Close (5) = 0
GetHostName (0X0FFFFFFFFFFF9C00, 512) = 0
Access ("/tmp/.oracle", 0) = 0
chmod ("/tmp/.oracle", 01777) err#1 eperm
Socket (1, 1, 0) = 5
Access ("/tmp/.oracle/slistener", 0) = 0
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused--> Note here, found to read/tmp/.oracle/slistener
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
_nsleep (0x0fffffffffff5580, 0x0fffffffffff5650) = 0
Close (5) = 0
Socket (1, 1, 0) = 5
Connext (5, 0X0FFFFFFFFFFF5BA8, 1025) err#79 econnrefused
Access ("/tmp/.oracle/slistener", 0) = 0
Close (5) = 0
Kopen ("/U01/APP/11.2.0.4/GRID/NETWORK/MESG/TNSUS.MSB", o_rdonly) = 5
Kfcntl (5, F_SETFD, 0x0000000000000001) = 0
Lseek (5, 0, 0) = 0
Kread (5, "1513" 011303\t\t\0\0\0\0 "..., 256) = 256
Lseek (5, 512, 0) = 512
Kread (5, "19b\0\0\0\0\0\0\0\0\0\0".., 512) = 512
Lseek (5, 1024, 0) = 1024
Kread (5, "\0\t\012\01a\0 &\0-\0 F".., 172) = 172
Lseek (5, 19456, 0) = 19456
Kread (5, "\00f04 $\0\0\0 b04%\0\0".., 512) = 512
Starting/u01/app/11.2.0.4/grid/bin/tnslsnr:please wait ...
KWrite (1, "S t a r t i n g/u 0" ..., 61) = 61
Kfcntl (1, F_GETFL, 0x0000000000000008) = 67110914
Pipe (0X0FFFFFFFFFFF80E0) = 0
Pipe (0X0FFFFFFFFFFF80D8) = 0
Sigprocmask (0, 0x09001000a090db70, 0x09001000a090db90) = 0
Kfork () = 19988596
Thread_setmystate (0x0fffffffffff7a70, 0x0000000000000000) = 0
Received signal #20, SIGCHLD [default]
Sigprocmask (0, 0x0fffffffffff7e48, 0x0000000000000000) = 0
Sigprocmask (2, 0x09001000a090db70, 0x0fffffffffff7bf0) = 0
_sigaction (0X0FFFFFFFFFFF7D20, 0x0fffffffffff7d78) = 0
Thread_setmystate (0x0fffffffffff77f0, 0x0000000000000000) = 0
Sigprocmask (1, 0x0fffffffffff7ec8, 0x0000000000000000) = 0
Kwaitpid (0x0fffffffffff8090, 19988596, 4, 0x0000000000000000, 0x0000000000000000) = 19988596
Close (6) = 0
Close (9) = 0
Kfcntl (8, f_setfd, 0x0000000000000001) = 0
Kfcntl (7, f_setfd, 0x0000000000000001) = 0
Kread (8, "N T P 0 1 4 4 8 3 5 7" ..., 64) = 14
_getpid () = 8912948
Kfcntl (8, f_setfd, 0x0000000000000001) = 0
KWrite (7, "\001\0\0\001:01," ..., 179) = 179
Kread (8, "\0\f\0\004\0\0\0" \001., 8208) = 492
Close (7) = 0
Close (8) = 0
Tnslsnr for Ibm/aix RISC system/6000:version 11.2.0.4.0-production
System parameter File Is/u01/app/11.2.0.4/grid/network/admin/listener.ora
Log messages Written To/u01/app/grid/diag/tnslsnr/xxxxdbb/listener/alert/log.xml
Error listening on: (Description= (Address= (PROTOCOL=IPC) (Key=listener))
Tns-12555:tns:permission denied
Tns-12560:tns:protocol Adapter Error
Tns-00525:insufficient Privilege for operation
Ibm/aix RISC system/6000 error:1: Not owner
KWrite (1, "T N s L s n r f o r" ..., 470) = 470
KWrite (1, "\ n", 1) = 1
Lseek (5, 19968, 0) = 19968
Kread (5, "\0\r04 5\0\0\0 V04 6\0\0".., 512) = 512
Listener failed to start. The error message (s) above ...
KWrite (1, "L i s t e n e r f a I"., 61) = 61
Kfcntl (1, F_GETFL, 0x0000000000000008) = 67110914
Close (4) = 0
Kfcntl (1, F_GETFL, 0x0000000000000008) = 67110914
Close (5) = 0
Kfcntl (1, F_GETFL, 0x0000000013f600ab) = 67110914
Kfcntl (2, F_GETFL, 0x0000000013f600ab) = 67110914
_exit (1)
Listen for normal boot after attempting to delete/tmp/.oracle/slistener