Dear PBS community,
I’m facing a similar problem to the one mentioned in Job gets stuck in a queue after a fresh install and Job stack in queue after fresh install | Permission error 15008. I still can’t resolve the issue by following the commends in these topics.
I installed the PBS in Centos 7.7 using openhpc tools. I was able to submit the jobs but they just stayed in queue.
/etc/pbs.conf:
PBS_EXEC=/opt/pbs
PBS_SERVER=agua
PBS_START_SERVER=1
PBS_START_SCHED=1
PBS_START_COMM=1
PBS_START_MOM=0
PBS_HOME=/var/spool/pbs
PBS_CORE_LIMIT=unlimited
PBS_SCP=/bin/scp
PBS_LEAF_NAME=agua
/etc/hosts:
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
10.0.2.1 agua
###ALL ENTRIES BELOW THIS LINE WILL BE OVERWRITTEN BY WAREWULF
#See provision.conf for configuration paramaters#Node Entry for node: n001 (ID=7)
10.0.2.2 n001.localdomain n001 n001-enp94s0f0.localdomain n001-enp94s0f0
/var/spool/pbs/mom_priv/config:
$clienthost agua
host resolvability:
[root@agua ~]# pbs_hostn -v agua
primary name: agua (from gethostbyname())
aliases: -none-
address length: 4 bytes
address: 10.0.2.1 (16908298 dec) name: agua
nmaps:
[root@agua ~]# nmap agua
Starting Nmap 6.40 ( http://nmap.org ) at 2021-11-23 04:02 EST
Nmap scan report for agua (10.0.2.1)
Host is up (0.0000080s latency).
Not shown: 991 closed ports
PORT STATE SERVICE
22/tcp open ssh
80/tcp open http
111/tcp open rpcbind
2049/tcp open nfs
3306/tcp open mysql
8649/tcp open unknown
8651/tcp open unknown
8652/tcp open unknown
15004/tcp open unknown
netstat -plunt:
[root@agua ~]# netstat -plunt
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:15001 0.0.0.0:* LISTEN 39034/pbs_server.bi
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 2318/master
tcp 0 0 0.0.0.0:15004 0.0.0.0:* LISTEN 38686/pbs_sched
tcp 0 0 0.0.0.0:15007 0.0.0.0:* LISTEN 39012/postgres
tcp 0 0 0.0.0.0:2049 0.0.0.0:* LISTEN -
tcp 0 0 0.0.0.0:17001 0.0.0.0:* LISTEN 38671/pbs_comm
tcp 0 0 0.0.0.0:8649 0.0.0.0:* LISTEN 13735/gmond
tcp 0 0 0.0.0.0:41609 0.0.0.0:* LISTEN -
tcp 0 0 0.0.0.0:3306 0.0.0.0:* LISTEN 13227/mysqld
tcp 0 0 0.0.0.0:8651 0.0.0.0:* LISTEN 13749/gmetad
tcp 0 0 0.0.0.0:8652 0.0.0.0:* LISTEN 13749/gmetad
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 13473/rpcbind
tcp 0 0 0.0.0.0:20048 0.0.0.0:* LISTEN 13475/rpc.mountd
tcp 0 0 0.0.0.0:47600 0.0.0.0:* LISTEN 13465/rpc.statd
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1969/sshd
tcp6 0 0 :::42968 :::* LISTEN -
tcp6 0 0 ::1:25 :::* LISTEN 2318/master
tcp6 0 0 :::15007 :::* LISTEN 39012/postgres
tcp6 0 0 :::2049 :::* LISTEN -
tcp6 0 0 :::111 :::* LISTEN 13473/rpcbind
tcp6 0 0 :::80 :::* LISTEN 13773/httpd
tcp6 0 0 :::20048 :::* LISTEN 13475/rpc.mountd
tcp6 0 0 :::45844 :::* LISTEN 13465/rpc.statd
tcp6 0 0 :::22 :::* LISTEN 1969/sshd
udp 0 0 0.0.0.0:20048 0.0.0.0:* 13475/rpc.mountd
udp 0 0 0.0.0.0:67 0.0.0.0:* 13969/dhcpd
udp 0 0 0.0.0.0:69 0.0.0.0:* 4200/xinetd
udp 0 0 0.0.0.0:111 0.0.0.0:* 13473/rpcbind
udp 0 0 10.0.2.1:123 0.0.0.0:* 3956/ntpd
udp 0 0 140.125.48.47:123 0.0.0.0:* 3956/ntpd
udp 0 0 127.0.0.1:123 0.0.0.0:* 3956/ntpd
udp 0 0 0.0.0.0:123 0.0.0.0:* 3956/ntpd
udp 0 0 0.0.0.0:514 0.0.0.0:* 13539/rsyslogd
udp 0 0 127.0.0.1:921 0.0.0.0:* 13465/rpc.statd
udp 0 0 0.0.0.0:927 0.0.0.0:* 13473/rpcbind
udp 0 0 0.0.0.0:2049 0.0.0.0:* -
udp 0 0 0.0.0.0:36182 0.0.0.0:* 13465/rpc.statd
udp 0 0 0.0.0.0:8649 0.0.0.0:* 13735/gmond
udp 0 0 0.0.0.0:41777 0.0.0.0:* -
udp6 0 0 :::20048 :::* 13475/rpc.mountd
udp6 0 0 :::111 :::* 13473/rpcbind
udp6 0 0 fe80::f592:7646:819:123 :::* 3956/ntpd
udp6 0 0 ::1:123 :::* 3956/ntpd
udp6 0 0 :::123 :::* 3956/ntpd
udp6 0 0 :::514 :::* 13539/rsyslogd
udp6 0 0 :::927 :::* 13473/rpcbind
udp6 0 0 :::2049 :::* -
udp6 0 0 :::43363 :::* -
udp6 0 0 :::46741 :::* 13465/rpc.statd
Based on the above info, I am not sure whether I have successfully turned on the Ports 15001 to 15007 and 17001. But these ports have been added to the ‘public’.
[root@agua ~]# firewall-cmd --zone=public --list-all
public (active)
target: default
icmp-block-inversion: no
interfaces: enp94s0f1
sources:
services: dhcpv6-client ssh
ports: 15001/tcp 15002/tcp 15003/tcp 15004/tcp 15005/tcp 15006/tcp 15007/tcp 15008/tcp 15009/tcp 17001/tcp
protocols:
masquerade: no
forward-ports:
source-ports:
icmp-blocks:
rich rules:
SELinux is disabled.
Firewalls are disabled.
PBS_SERVER is resolvable.
server_logs:
11/23/2021 04:15:25;0040;Server@agua;Svr;agua;Scheduler sent command 3
11/23/2021 04:15:25;0040;Server@agua;Svr;agua;Scheduler sent command 0
11/23/2021 04:15:25;0080;Server@agua;Req;?;Req Header bad, errno 104, dis error 7
11/23/2021 04:15:25;0080;Server@agua;Req;req_reject;Reject reply code=15056, aux=0, type=0, from @agua
11/23/2021 04:15:25;0080;Server@agua;Req;?;Req Header bad, errno 104, dis error 7
11/23/2021 04:15:25;0080;Server@agua;Req;req_reject;Reject reply code=15056, aux=0, type=0, from @agua
11/23/2021 04:16:02;0100;Server@agua;Req;;Type 0 request received from test@agua, sock=15
11/23/2021 04:16:02;0100;Server@agua;Req;;Type 49 request received from test@agua, sock=16
11/23/2021 04:16:02;0100;Server@agua;Req;;Type 21 request received from test@agua, sock=15
11/23/2021 04:16:02;0100;Server@agua;Req;;Type 1 request received from test@agua, sock=15
11/23/2021 04:16:02;0100;Server@agua;Req;;Type 3 request received from test@agua, sock=15
11/23/2021 04:16:02;0100;Server@agua;Req;;Type 5 request received from test@agua, sock=15
11/23/2021 04:16:02;0100;Server@agua;Job;4.agua;enqueuing into workq, state 1 hop 1
11/23/2021 04:16:02;0008;Server@agua;Job;4.agua;Job Queued at request of test@agua, owner = test@agua, job name = test, queue = workq
11/23/2021 04:16:02;0040;Server@agua;Svr;agua;Scheduler sent command 1
11/23/2021 04:16:02;0040;Server@agua;Svr;agua;Scheduler sent command 0
11/23/2021 04:16:02;0080;Server@agua;Req;?;Req Header bad, errno 104, dis error 7
11/23/2021 04:16:02;0080;Server@agua;Req;req_reject;Reject reply code=15056, aux=0, type=0, from @agua
11/23/2021 04:16:02;0080;Server@agua;Req;?;Req Header bad, errno 104, dis error 7
11/23/2021 04:16:02;0080;Server@agua;Req;req_reject;Reject reply code=15056, aux=0, type=0, from @agua
11/23/2021 04:16:08;0100;Server@agua;Req;;Type 0 request received from test@agua, sock=15
11/23/2021 04:16:08;0100;Server@agua;Req;;Type 49 request received from test@agua, sock=16
11/23/2021 04:16:08;0100;Server@agua;Req;;Type 21 request received from test@agua, sock=15
11/23/2021 04:16:08;0100;Server@agua;Req;;Type 51 request received from test@agua, sock=15
11/23/2021 04:16:08;0100;Server@agua;Req;;Type 0 request received from test@agua, sock=16
11/23/2021 04:16:08;0100;Server@agua;Req;;Type 49 request received from test@agua, sock=17
11/23/2021 04:16:08;0100;Server@agua;Req;;Type 21 request received from test@agua, sock=16
There’s an error message with code 15056.
11/23/2021 04:16:02;0080;Server@agua;Req;req_reject;Reject reply code=15056, aux=0, type=0, from @agua
sched_logs:
11/23/2021 00:09:25;0002;pbs_sched;Svr;Log;Log opened
11/23/2021 00:09:25;0002;pbs_sched;Svr;pbs_sched;pbs_version=19.1.3
11/23/2021 00:09:25;0002;pbs_sched;Svr;pbs_sched;pbs_build=mach=N/A:security=N/A:configure_args=N/A
11/23/2021 00:09:25;0002;pbs_sched;Svr;pbs_sched;hostname=140.125.48.45;pbs_leaf_name=agua;pbs_mom_node_name=N/A
11/23/2021 00:09:25;0002;pbs_sched;Svr;pbs_sched;ipv4 interface lo: localhost4.localdomain4
11/23/2021 00:09:25;0002;pbs_sched;Svr;pbs_sched;ipv4 interface enp94s0f0: agua
11/23/2021 00:09:25;0002;pbs_sched;Svr;pbs_sched;ipv4 interface enp94s0f1: agua.yuntech.edu.tw
11/23/2021 00:09:25;0002;pbs_sched;Svr;pbs_sched;ipv6 interface lo: localhost6.localdomain6
11/23/2021 00:09:25;0002;pbs_sched;Svr;pbs_sched;ipv6 interface enp94s0f1: agua.yuntech.edu.tw
11/23/2021 00:09:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 884 unauthorized host
11/23/2021 00:09:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 883 unauthorized host
11/23/2021 00:19:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 882 unauthorized host
11/23/2021 00:19:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 881 unauthorized host
11/23/2021 00:29:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 880 unauthorized host
11/23/2021 00:29:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 879 unauthorized host
11/23/2021 00:39:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 878 unauthorized host
11/23/2021 00:39:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 877 unauthorized host
11/23/2021 00:49:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 876 unauthorized host
11/23/2021 00:49:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 875 unauthorized host
11/23/2021 00:59:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 874 unauthorized host
11/23/2021 00:59:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 873 unauthorized host
11/23/2021 01:09:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 872 unauthorized host
11/23/2021 01:09:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 871 unauthorized host
11/23/2021 01:19:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 870 unauthorized host
11/23/2021 01:19:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 869 unauthorized host
11/23/2021 01:29:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 868 unauthorized host
11/23/2021 01:29:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 867 unauthorized host
11/23/2021 01:39:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 866 unauthorized host
11/23/2021 01:39:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 865 unauthorized host
11/23/2021 01:49:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 864 unauthorized host
11/23/2021 01:49:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 863 unauthorized host
11/23/2021 01:59:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 862 unauthorized host
11/23/2021 01:59:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 861 unauthorized host
11/23/2021 02:09:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 860 unauthorized host
11/23/2021 02:09:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 859 unauthorized host
11/23/2021 02:19:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 858 unauthorized host
11/23/2021 02:19:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 857 unauthorized host
11/23/2021 02:29:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 856 unauthorized host
11/23/2021 02:29:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 855 unauthorized host
11/23/2021 02:39:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 854 unauthorized host
11/23/2021 02:39:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 853 unauthorized host
11/23/2021 02:48:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 852 unauthorized host
11/23/2021 02:48:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 851 unauthorized host
11/23/2021 02:58:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 850 unauthorized host
11/23/2021 02:58:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 849 unauthorized host
11/23/2021 03:08:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 848 unauthorized host
11/23/2021 03:08:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 847 unauthorized host
11/23/2021 03:18:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 846 unauthorized host
11/23/2021 03:18:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 845 unauthorized host
11/23/2021 03:25:47;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 60504 non-reserved port
11/23/2021 03:28:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 844 unauthorized host
11/23/2021 03:28:58;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 843 unauthorized host
11/23/2021 03:35:17;0002;pbs_sched;Svr;die;caught signal 15
11/23/2021 03:35:17;0002;pbs_sched;Svr;Log;Log closed
11/23/2021 03:35:18;0002;pbs_sched;Svr;Log;Log opened
11/23/2021 03:35:18;0002;pbs_sched;Svr;pbs_sched;pbs_version=19.1.3
11/23/2021 03:35:18;0002;pbs_sched;Svr;pbs_sched;pbs_build=mach=N/A:security=N/A:configure_args=N/A
11/23/2021 03:35:18;0002;pbs_sched;Svr;pbs_sched;hostname=140.125.48.45;pbs_leaf_name=agua;pbs_mom_node_name=N/A
11/23/2021 03:35:18;0002;pbs_sched;Svr;pbs_sched;ipv4 interface lo: localhost4.localdomain4
11/23/2021 03:35:18;0002;pbs_sched;Svr;pbs_sched;ipv4 interface enp94s0f0: agua
11/23/2021 03:35:18;0002;pbs_sched;Svr;pbs_sched;ipv4 interface enp94s0f1: agua.yuntech.edu.tw
11/23/2021 03:35:18;0002;pbs_sched;Svr;pbs_sched;ipv6 interface lo: localhost6.localdomain6
11/23/2021 03:35:18;0002;pbs_sched;Svr;pbs_sched;ipv6 interface enp94s0f1: agua.yuntech.edu.tw
11/23/2021 03:35:18;0002;pbs_sched;n/a;setup_env;read environment from /var/spool/pbs/pbs_environment
11/23/2021 03:35:18;0040;pbs_sched;Fil;sched_config;Obsolete config name sort_queues
11/23/2021 03:35:18;0004;pbs_sched;Fil;holidays;The holiday file is out of date; please update it.
11/23/2021 03:35:18;0040;pbs_sched;Fil;fairshare usage;Creating usage database for fairshare
11/23/2021 03:35:18;0006;pbs_sched;Fil;pbs_sched;Version 19.1.3, started, initialization type = 0
11/23/2021 03:35:18;0002;pbs_sched;Svr;main;/opt/pbs/sbin/pbs_sched startup pid 38686
11/23/2021 03:35:18;0d80;pbs_sched;TPP;pbs_sched(Main Thread);TPP set to use reserved port authentication
11/23/2021 03:35:18;0c06;pbs_sched;TPP;pbs_sched(Main Thread);TPP leaf node names = agua:15004
11/23/2021 03:35:18;0d80;pbs_sched;TPP;pbs_sched(Main Thread);Initializing TPP transport Layer
11/23/2021 03:35:18;0d80;pbs_sched;TPP;pbs_sched(Main Thread);Max files allowed = 1024
11/23/2021 03:35:18;0c06;pbs_sched;TPP;pbs_sched(Main Thread);Max files too low - you may want to increase it.
11/23/2021 03:35:18;0d80;pbs_sched;TPP;pbs_sched(Main Thread);TPP initialization done
11/23/2021 03:35:18;0c06;pbs_sched;TPP;pbs_sched(Main Thread);Single pbs_comm configured, TPP Fault tolerant mode disabled
11/23/2021 03:35:18;0d80;pbs_sched;TPP;pbs_sched(Main Thread);Connecting to pbs_comm agua:17001
11/23/2021 03:35:18;0c06;pbs_sched;TPP;pbs_sched(Thread 0);Thread ready
11/23/2021 03:35:18;0c06;pbs_sched;TPP;pbs_sched(Thread 0);Registering address 10.0.2.1:15004 to pbs_comm
11/23/2021 03:35:18;0c06;pbs_sched;TPP;pbs_sched(Thread 0);Connected to pbs_comm agua:17001
11/23/2021 03:35:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 634 unauthorized host
11/23/2021 03:35:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 633 unauthorized host
11/23/2021 03:35:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 632 unauthorized host
11/23/2021 03:35:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 631 unauthorized host
11/23/2021 03:38:15;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 34034 non-reserved port
11/23/2021 03:45:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 630 unauthorized host
11/23/2021 03:45:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 629 unauthorized host
11/23/2021 03:55:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 628 unauthorized host
11/23/2021 03:55:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 627 unauthorized host
11/23/2021 04:05:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 626 unauthorized host
11/23/2021 04:05:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 625 unauthorized host
11/23/2021 04:15:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 624 unauthorized host
11/23/2021 04:15:25;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 623 unauthorized host
11/23/2021 04:16:02;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 622 unauthorized host
11/23/2021 04:16:02;0001;pbs_sched;Svr;pbs_sched;badconn, agua on port 621 unauthorized host
Not sure where I made the mistakes. Any help is appreciated!! THANKS!!