# PBS load-balancing issues across 4 execution queues

**URL:** <https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410>\
**Category:** Users/Site Administrators\
**Created:** [December 19, 2020, 1:23am UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410 "2020-12-19T01:23:31Z")\
**Posts on this page:** 9\
**Page:** 1

<div class="post-metadata">

**Author:** ![avakhnin](https://avatars.discourse-cdn.com/v4/letter/a/77aa72/32.png) [@avakhnin](https://community.openpbs.org/u/avakhnin)\
**Post date:** [December 19, 2020, 1:23am UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410/1 "2020-12-19T01:23:32Z")

</div>

Hello,

We have PBS cluster with a route queue and 4 execution queues. The load\_balancing: true ALL in /var/spool/pbs/sched\_priv/sched\_config but jobs only run in 2 execution queues (xeon1700s/w). No jobs are running in xeon1800s/w execution queues. Please help to detect configuration issue. We have PBS Pro 14.0 installed in the environment.

The goal is to loadbalance jobs across xeon1700s/xeon1800s and xeon1700w/xeon1800w based on how many CPU requested per job

Route Queue - xeon1600  
create queue xeon1600  
set queue xeon1600 queue\_type = Route  
set queue xeon1600 route\_destinations = “xeon1700s,xeon1800s,xeon1700w,xeon1800w”  
set queue xeon1600 enabled = True  
set queue xeon1600 started = True

create queue xeon1700s  
set queue xeon1700s queue\_type = Execution  
set queue xeon1700s resources\_max.ncpus = 80   
set queue xeon1700s resources\_min.ncpus = 1  
set queue xeon1700s enabled = True  
set queue xeon1700s started = True

create queue xeon1700w  
set queue xeon1700w queue\_type = Execution  
set queue xeon1700w resources\_max.ncpus = 4200  
set queue xeon1700w resources\_min.ncpus = 800  
set queue xeon1700w enabled = True  
set queue xeon1700w started = True

create queue xeon1800s  
set queue xeon1800s queue\_type = Execution  
set queue xeon1800s resources\_min.ncpus = 1  
set queue xeon1800s resources\_max.ncpus = 80  
set queue xeon1800s enabled = True  
set queue xeon1800s started = True

create queue xeon1800w  
set queue xeon1800w queue\_type = Execution  
set queue xeon1800w resources\_min.ncpus = 800  
set queue xeon1700w resources\_max.ncpus = 6280  
set queue xeon1800w enabled = True  
set queue xeon1800w started = True

 ![Screen Shot 2020-12-18 at 8.00.35 PM](https://canada1.discourse-cdn.com/flex030/uploads/pbspro/original/1X/92ef529aeb4b312d13b8932bd4af8cb9df3b588a.png)

qmgr config output file

# Create resources and set their properties.

# 

# 

# Create and define resource qlist

# 

create resource qlist  
set resource qlist type = string\_array  
set resource qlist flag = h

# 

# Create and define resource acfd\_fluent\_solver\_lic

# 

create resource acfd\_fluent\_solver\_lic  
set resource acfd\_fluent\_solver\_lic type = long

# 

# Create and define resource acfd\_cfx\_solver\_lic

# 

create resource acfd\_cfx\_solver\_lic  
set resource acfd\_cfx\_solver\_lic type = long

# 

# Create and define resource acfd\_par\_proc\_lic

# 

create resource acfd\_par\_proc\_lic  
set resource acfd\_par\_proc\_lic type = long

# 

# Create queues and set their attributes.

# 

# 

# Create and define queue xeon1300g

# 

create queue xeon1300g  
set queue xeon1300g queue\_type = Execution  
set queue xeon1300g resources\_max.qlist = xeon1300g  
set queue xeon1300g resources\_min.qlist = xeon1300g  
set queue xeon1300g resources\_default.qlist = xeon1300g  
set queue xeon1300g default\_chunk.qlist = xeon1300g  
set queue xeon1300g enabled = True  
set queue xeon1300g started = True

# 

# Create and define queue xeon1400g

# 

create queue xeon1400g  
set queue xeon1400g queue\_type = Execution  
set queue xeon1400g resources\_max.qlist = xeon1400g  
set queue xeon1400g resources\_min.qlist = xeon1400g  
set queue xeon1400g resources\_default.qlist = xeon1400g  
set queue xeon1400g default\_chunk.qlist = xeon1400g  
set queue xeon1400g enabled = True  
set queue xeon1400g started = True

# 

# Create and define queue xeon1300

# 

create queue xeon1300  
set queue xeon1300 queue\_type = Execution  
set queue xeon1300 resources\_max.qlist = xeon1300  
set queue xeon1300 resources\_min.qlist = xeon1300  
set queue xeon1300 resources\_default.qlist = xeon1300  
set queue xeon1300 default\_chunk.qlist = xeon1300  
set queue xeon1300 enabled = True  
set queue xeon1300 started = True

# 

# Create and define queue xeon10g11g

# 

create queue xeon10g11g  
set queue xeon10g11g queue\_type = Execution  
set queue xeon10g11g resources\_max.qlist = xeon10g11g  
set queue xeon10g11g resources\_min.qlist = xeon10g11g  
set queue xeon10g11g resources\_default.qlist = xeon10g11g  
set queue xeon10g11g default\_chunk.qlist = xeon10g11g  
set queue xeon10g11g enabled = True  
set queue xeon10g11g started = True

# 

# Create and define queue xeon1500

# 

create queue xeon1500  
set queue xeon1500 queue\_type = Execution  
set queue xeon1500 resources\_max.qlist = xeon1500  
set queue xeon1500 resources\_min.qlist = xeon1500  
set queue xeon1500 resources\_default.qlist = xeon1500  
set queue xeon1500 default\_chunk.qlist = xeon1500  
set queue xeon1500 enabled = True  
set queue xeon1500 started = True

# 

# Create and define queue xeon1400

# 

create queue xeon1400  
set queue xeon1400 queue\_type = Execution  
set queue xeon1400 resources\_max.qlist = xeon1400  
set queue xeon1400 resources\_min.qlist = xeon1400  
set queue xeon1400 resources\_default.qlist = xeon1400  
set queue xeon1400 default\_chunk.qlist = xeon1400  
set queue xeon1400 enabled = True  
set queue xeon1400 started = True

# 

# Create and define queue xeon10pri

# 

create queue xeon10pri  
set queue xeon10pri queue\_type = Execution  
set queue xeon10pri resources\_max.qlist = xeon10pri  
set queue xeon10pri resources\_min.qlist = xeon10pri  
set queue xeon10pri resources\_default.qlist = xeon10pri  
set queue xeon10pri default\_chunk.qlist = xeon10pri  
set queue xeon10pri enabled = True  
set queue xeon10pri started = True

# 

# Create and define queue xeon900

# 

create queue xeon900  
set queue xeon900 queue\_type = Execution  
set queue xeon900 resources\_max.qlist = xeon900  
set queue xeon900 resources\_min.qlist = xeon900  
set queue xeon900 resources\_default.qlist = xeon900  
set queue xeon900 default\_chunk.qlist = xeon900  
set queue xeon900 enabled = True  
set queue xeon900 started = True

# 

# Create and define queue xeon1700

# 

create queue xeon1700  
set queue xeon1700 queue\_type = Execution  
set queue xeon1700 resources\_max.qlist = xeon1700  
set queue xeon1700 resources\_min.qlist = xeon1700  
set queue xeon1700 resources\_default.qlist = xeon1700  
set queue xeon1700 default\_chunk.qlist = xeon1700  
set queue xeon1700 enabled = False  
set queue xeon1700 started = False

# 

# Create and define queue xeon1200g-1400g

# 

create queue xeon1200g-1400g  
set queue xeon1200g-1400g queue\_type = Execution  
set queue xeon1200g-1400g resources\_max.qlist = xeon1200g-1400g  
set queue xeon1200g-1400g resources\_min.qlist = xeon1200g-1400g  
set queue xeon1200g-1400g resources\_default.qlist = xeon1200g-1400g  
set queue xeon1200g-1400g default\_chunk.qlist = xeon1200g-1400g  
set queue xeon1200g-1400g enabled = True  
set queue xeon1200g-1400g started = True

# 

# Create and define queue xeon1800

# 

create queue xeon1800  
set queue xeon1800 queue\_type = Execution  
set queue xeon1800 resources\_max.qlist = xeon1800  
set queue xeon1800 resources\_min.qlist = xeon1800  
set queue xeon1800 resources\_default.qlist = xeon1800  
set queue xeon1800 default\_chunk.qlist = xeon1800  
set queue xeon1800 enabled = False  
set queue xeon1800 started = False

# 

# Create and define queue xeon1000

# 

create queue xeon1000  
set queue xeon1000 queue\_type = Execution  
set queue xeon1000 resources\_max.qlist = xeon1000  
set queue xeon1000 resources\_min.qlist = xeon1000  
set queue xeon1000 resources\_default.qlist = xeon1000  
set queue xeon1000 default\_chunk.qlist = xeon1000  
set queue xeon1000 enabled = True  
set queue xeon1000 started = True

# 

# Create and define queue xeon1100

# 

create queue xeon1100  
set queue xeon1100 queue\_type = Execution  
set queue xeon1100 resources\_max.qlist = xeon1100  
set queue xeon1100 resources\_min.qlist = xeon1100  
set queue xeon1100 resources\_default.qlist = xeon1100  
set queue xeon1100 default\_chunk.qlist = xeon1100  
set queue xeon1100 enabled = True  
set queue xeon1100 started = True

# 

# Create and define queue xeon1100g

# 

create queue xeon1100g  
set queue xeon1100g queue\_type = Execution  
set queue xeon1100g resources\_max.qlist = xeon1100g  
set queue xeon1100g resources\_min.qlist = xeon1100g  
set queue xeon1100g resources\_default.qlist = xeon1100g  
set queue xeon1100g default\_chunk.qlist = xeon1100g  
set queue xeon1100g enabled = True  
set queue xeon1100g started = True

# 

# Create and define queue xeon1200

# 

create queue xeon1200  
set queue xeon1200 queue\_type = Execution  
set queue xeon1200 resources\_max.qlist = xeon1200  
set queue xeon1200 resources\_min.qlist = xeon1200  
set queue xeon1200 resources\_default.qlist = xeon1200  
set queue xeon1200 default\_chunk.qlist = xeon1200  
set queue xeon1200 enabled = True  
set queue xeon1200 started = True

# 

# Create and define queue xeon1200g

# 

create queue xeon1200g  
set queue xeon1200g queue\_type = Execution  
set queue xeon1200g resources\_max.qlist = xeon1200g  
set queue xeon1200g resources\_min.qlist = xeon1200g  
set queue xeon1200g resources\_default.qlist = xeon1200g  
set queue xeon1200g default\_chunk.qlist = xeon1200g  
set queue xeon1200g enabled = True  
set queue xeon1200g started = True

# 

# Create and define queue xeon1000g

# 

create queue xeon1000g  
set queue xeon1000g queue\_type = Execution  
set queue xeon1000g resources\_max.qlist = xeon1000g  
set queue xeon1000g resources\_min.qlist = xeon1000g  
set queue xeon1000g resources\_default.qlist = xeon1000g  
set queue xeon1000g default\_chunk.qlist = xeon1000g  
set queue xeon1000g enabled = True  
set queue xeon1000g started = True

# 

# Create and define queue xeon1700s

# 

create queue xeon1700s  
set queue xeon1700s queue\_type = Execution  
set queue xeon1700s resources\_max.ncpus = 80  
set queue xeon1700s resources\_max.qlist = xeon1700s  
set queue xeon1700s resources\_min.ncpus = 1  
set queue xeon1700s resources\_min.qlist = xeon1700s  
set queue xeon1700s resources\_default.qlist = xeon1700s  
set queue xeon1700s default\_chunk.qlist = xeon1700s  
set queue xeon1700s enabled = True  
set queue xeon1700s started = True

# 

# Create and define queue xeon1700w

# 

create queue xeon1700w  
set queue xeon1700w queue\_type = Execution  
set queue xeon1700w resources\_max.ncpus = 6280  
set queue xeon1700w resources\_max.qlist = xeon1700w  
set queue xeon1700w resources\_min.ncpus = 81  
set queue xeon1700w resources\_min.qlist = xeon1700w  
set queue xeon1700w resources\_default.qlist = xeon1700w  
set queue xeon1700w default\_chunk.qlist = xeon1700w  
set queue xeon1700w enabled = True  
set queue xeon1700w started = True

# 

# Create and define queue xeon1800s

# 

create queue xeon1800s  
set queue xeon1800s queue\_type = Execution  
set queue xeon1800s resources\_max.ncpus = 80  
set queue xeon1800s resources\_max.qlist = xeon1800s  
set queue xeon1800s resources\_min.ncpus = 1  
set queue xeon1800s resources\_min.qlist = xeon1800s  
set queue xeon1800s resources\_default.qlist = xeon1800s  
set queue xeon1800s default\_chunk.qlist = xeon1800s  
set queue xeon1800s max\_user\_run = 5  
set queue xeon1800s enabled = True  
set queue xeon1800s started = True

# 

# Create and define queue xeon1800w

# 

create queue xeon1800w  
set queue xeon1800w queue\_type = Execution  
set queue xeon1800w resources\_max.qlist = xeon1800w  
set queue xeon1800w resources\_min.ncpus = 81  
set queue xeon1800w resources\_min.qlist = xeon1800w  
set queue xeon1800w resources\_default.qlist = xeon1800w  
set queue xeon1800w default\_chunk.qlist = xeon1800w  
set queue xeon1800w max\_user\_run = 5  
set queue xeon1800w enabled = True  
set queue xeon1800w started = True

# 

# Create and define queue xeon1600

# 

create queue xeon1600  
set queue xeon1600 queue\_type = Route  
set queue xeon1600 route\_destinations = xeon1700s  
set queue xeon1600 route\_destinations += xeon1800s  
set queue xeon1600 route\_destinations += xeon1700w  
set queue xeon1600 route\_destinations += xeon1800w  
set queue xeon1600 enabled = True  
set queue xeon1600 started = True

# 

# Set server attributes.

# 

set server scheduling = True  
set server default\_queue = xeon1600  
set server log\_events = 511  
set server mail\_from = adm  
set server query\_other\_jobs = True  
set server resources\_default.ncpus = 1  
set server default\_chunk.ncpus = 1  
set server scheduler\_iteration = 600  
set server resv\_enable = True  
set server node\_fail\_requeue = 310  
set server max\_array\_size = 10000  
set server pbs\_license\_min = 0  
set server pbs\_license\_max = 2147483647  
set server pbs\_license\_linger\_time = 31536000  
set server license\_count = Avail\_Global:1000000 Avail\_Local:1000000 Used:0 High\_Use:0 Avail\_Sockets:1000000 Unused\_Sockets:1000000  
set server eligible\_time\_enable = False  
set server max\_concurrent\_provision = 5

---

<div class="post-metadata">

**Author:** ![adarsh](https://avatars.discourse-cdn.com/v4/letter/a/f07891/32.png) [@adarsh](https://community.openpbs.org/u/adarsh)\
**Post date:** [December 19, 2020, 11:59pm UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410/2 "2020-12-19T23:59:06Z")

</div>

Thank you for the above details.

Routing queues would be used only to route the request to specific queues based on what user has requested and sometimes to avoid users directly submitting to execution queues. Routing queues would not be satisfy load balancing. load\_balancing would be based on the load of the compute nodes ( many mom\_priv/config parameters with respect load are considered). You might have to look at Peer Scheduling, that might help.

Otherwise, please read the below

Please check the PBS Pro Administrator guide at this link: [https://www.altair.com/pdfs/pbsworks/PBSAdminGuide2020.1.pdf](https://www.altair.com/pdfs/pbsworks/PBSAdminGuide2020.1.pdf)

 ![image](https://canada1.discourse-cdn.com/flex030/uploads/pbspro/original/1X/b351218e70e583122bf36bb50a65694575f4e416.png)

I think you could implement the below:

 ![image](https://canada1.discourse-cdn.com/flex030/uploads/pbspro/original/1X/8a5b0ec5f50e1878f7c99a1f4d5fd862ff9ba898.png)

FYI: i know you have mentioned that your setup is on PBS Pro 14.0

The load\_balancing scheduler parameter (deprecated as of version 2020.1) controls a behavior wherein a scheduler won’t place a job where the anticipated load would exceed $max\_load. For example if a machine has a load of 1.25, is running a 1-CPU job, and has 2 CPUs, PBS won’t place another 1-CPU job there.

---

<div class="post-metadata">

**Author:** ![avakhnin](https://avatars.discourse-cdn.com/v4/letter/a/77aa72/32.png) [@avakhnin](https://community.openpbs.org/u/avakhnin)\
**Post date:** [December 21, 2020, 4:20am UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410/3 "2020-12-21T04:20:05Z")

</div>

Hello,

Okey, I see. the load\_balancing won’t help us to achieve a goal of distributing workload evenly across 4 slurm execution queue.

- Can we use peer scheduling in single slurm cluster, or have to separate resource in multiple clusters?
- Is there a way to evenly distribute workload across 4 execution queues using vnodes? Do we “create resource aveload type=” for each execution queue?

Best regards,

Andrei

---

<div class="post-metadata">

**Author:** ![adarsh](https://avatars.discourse-cdn.com/v4/letter/a/f07891/32.png) [@adarsh](https://community.openpbs.org/u/adarsh)\
**Post date:** [December 21, 2020, 8:00pm UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410/4 "2020-12-21T20:00:46Z")

</div>

I hope you meant PBS Pro OSS. The below would suffice your requirement

- peer scheduling
- sorting vnodes as per the load average (as per the 4.9.50.3)

Thank you

---

<div class="post-metadata">

**Author:** ![avakhnin](https://avatars.discourse-cdn.com/v4/letter/a/77aa72/32.png) [@avakhnin](https://community.openpbs.org/u/avakhnin)\
**Post date:** [December 22, 2020, 4:26am UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410/5 "2020-12-22T04:26:54Z")

</div>

Hi, yes sorry PBS Pro OSS. Thank you for recommendations. I will try to implement vnodes and peer scheduling on our test system

---

<div class="post-metadata">

**Author:** ![avakhnin](https://avatars.discourse-cdn.com/v4/letter/a/77aa72/32.png) [@avakhnin](https://community.openpbs.org/u/avakhnin)\
**Post date:** [December 27, 2020, 3:58am UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410/6 "2020-12-27T03:58:50Z")

</div>

Hi,

Looks like peer scheduling leverage multiple clusters/execution systems when resource are available, but this is not true load balancing. How can we achieve true loadlbalancing across 4 execution queues when jobs submitted in the default route queue?

“Peer to Peer” scheduling: A site can have multiple PBS  
Pro clusters (each cluster has its server, scheduler and one  
or more execution systems). A scheduler in any given  
cluster can be configured to move jobs from other clusters  
to its cluster when the resources required by the job are  
available locally

Thanks in advance

---

<div class="post-metadata">

**Author:** ![avakhnin](https://avatars.discourse-cdn.com/v4/letter/a/77aa72/32.png) [@avakhnin](https://community.openpbs.org/u/avakhnin)\
**Post date:** [December 27, 2020, 4:53am UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410/7 "2020-12-27T04:53:47Z")

</div>

According to the guide, the routing queue should try destinations in round-robin fashion, in the order listed, but this jobs only being routed to xeon1700s/w queues.

#Create and define queue xeon1600  
create queue xeon1600  
set queue xeon1600 queue\_type = Route  
set queue xeon1600 route\_destinations = xeon1700s  
set queue xeon1600 route\_destinations += xeon1800s  
set queue xeon1600 route\_destinations += xeon1700w  
set queue xeon1600 route\_destinations += xeon1800w  
set queue xeon1600 enabled = True  
set queue xeon1600 started = True

2.3.6 Routing Queues A routing queue is used only to route jobs; jobs cannot run from a routing queue. A routing queue has the following properties:  
• Can route to multiple destinations  
• Tries destinations in round-robin fashion, in the order listed  
• Can route to execution queues  
• Can route to other routing queues  
• Can route to queues in other complexes (at other servers)

Destinations can be specified in the following ways:  
route\_destinations = Q1  
route\_destinations = Q1@Server1  
route\_destinations = “Q1, Q2@Server1, Q3@Server2”  
route\_destinations += Q1  
route\_destinations += "Q4, Q5@Server3”

---

<div class="post-metadata">

**Author:** ![adarsh](https://avatars.discourse-cdn.com/v4/letter/a/f07891/32.png) [@adarsh](https://community.openpbs.org/u/adarsh)\
**Post date:** [December 29, 2020, 7:58pm UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410/8 "2020-12-29T19:58:41Z")

</div>

Could you please share your qsub statements along with your assessment of which queue should have received it based on the above routing/execution queue configuration.

---

<div class="post-metadata">

**Author:** ![avakhnin](https://avatars.discourse-cdn.com/v4/letter/a/77aa72/32.png) [@avakhnin](https://community.openpbs.org/u/avakhnin)\
**Post date:** [February 5, 2021, 4:17am UTC](https://community.openpbs.org/t/pbs-load-balancing-issues-across-4-execution-queues/2410/9 "2021-02-05T04:17:54Z")

</div>

Qsub statement

```
### Job Name
#PBS -N mpi_job
### Project code
#PBS -A MPI_Test
#PBS -l walltime=00:06:16
#PBS -q xeon1600
### Merge output and error files
#PBS -j oe
#PBS -k eod
### Select 2 nodes with 2 CPUs 
#PBS -lselect=4:ncpus=10:mpiprocs=16:switch=10gBE
### Send email on abort, begin and end
#PBS -m abe
### Specify mail recipient
##PBS -M john.fowler@xd-lab.net 
export I_MPI_FABRICS=ofi:tcp 
export I_MPI_STATS=ipm
echo "Start" 
cd ~/tpbs
date
ls 
date
echo "MPIRUN"
# mpirun -n 2 -iface eno1 -verbose IMB-P2P
# mpirun -iface eno1 -verbose IMB-P2P
mpirun -l -v -n 160 IMB-MPI1
echo "Sleep" 
sleep 300
echo "End" 

```

We have round\_robin: True all and expect jobs roundrobin across 2 execution queues

```
/var/spool/pbs/sched_priv/sched_config
backfill_prime:	false	ALL
by_queue: True non_prime
by_queue: True prime
dedicated_prefix: ded
fairshare_decay_factor: 0.5
fairshare_decay_time: 24:00:00
fairshare_entity: euser
fair_share: false	ALL
fairshare_usage_res: cput
help_starving_jobs:	true	ALL
max_starve: 24:00:00
node_sort_key: "sort_priority HIGH"	ALL
nonprimetime_prefix: np_
preemptive_sched: true	ALL
prime_exempt_anytime_queues:	false
primetime_prefix: p_
provision_policy: "aggressive_provision"
resources: "ncpus, mem, arch, host, vnode, aoe, eoe, Qlist, acfd_fluent_solver_lic, acfd_cfx_solver_lic, acfd_par_proc_lic"
round_robin: True	all
smp_cluster_dist: pack

$ qstat
Job id Name User Time Use S Queue
---------------- ---------------- ---------------- -------- - -----
197.lssd530-hs05 mpi-test.sh andrei 00:00:00 R xeon1800s       
198.lssd530-hs05 mpi-test.sh andrei 00:00:00 R xeon1800s       
199.lssd530-hs05 mpi-test.sh andrei 00:00:00 R xeon1800s       
200.lssd530-hs05 mpi-test.sh andrei 00:00:00 R xeon1800s       
201.lssd530-hs05 mpi-test.sh andrei 00:00:00 R xeon1800s       
202.lssd530-hs05 mpi-test.sh andrei 0 Q xeon1800s       
203.lssd530-hs05 t.sh john 0 Q xeon1800s       
204.lssd530-hs05 t.sh john 0 Q xeon1800s       
205.lssd530-hs05 t.sh john 0 Q xeon1800s       
206.lssd530-hs05 t.sh john 0 Q xeon1800s 

$ pbsnodes -a
lssd530-cs10
     Mom = lssd530-cs10
     ntype = PBS
     state = free
     pcpus = 104
     resources_available.arch = linux
     resources_available.host = lssd530-cs10
     resources_available.mem = 395571708kb
     resources_available.ncpus = 104
     resources_available.Qlist = xeon1700w,xeon1700s
     resources_available.vnode = lssd530-cs10
     resources_assigned.accelerator_memory = 0kb
     resources_assigned.hbmem = 0kb
     resources_assigned.mem = 0kb
     resources_assigned.naccelerators = 0
     resources_assigned.ncpus = 0
     resources_assigned.vmem = 0kb
     queue = xeon1700s
     resv_enable = True
     sharing = default_shared
     last_state_change_time = Thu Feb 4 02:52:08 2021
     last_used_time = Thu Feb 4 02:50:14 2021

lssd530-cs12
     Mom = lssd530-cs12
     ntype = PBS
     state = free
     pcpus = 104
     jobs = 202.lssd530-hs05/0, 202.lssd530-hs05/1, 202.lssd530-hs05/2, 202.lssd530-hs05/3, 202.lssd530-hs05/4, 202.lssd530-hs05/5, 202.lssd530-hs05/6, 202.lssd530-hs05/7, 202.lssd530-hs05/8, 202.lssd530-hs05/9, 202.lssd530-hs05/10, 202.lssd530-hs05/11, 202.lssd530-hs05/12, 202.lssd530-hs05/13, 202.lssd530-hs05/14, 202.lssd530-hs05/15, 202.lssd530-hs05/16, 202.lssd530-hs05/17, 202.lssd530-hs05/18, 202.lssd530-hs05/19, 202.lssd530-hs05/20, 202.lssd530-hs05/21, 202.lssd530-hs05/22, 202.lssd530-hs05/23, 202.lssd530-hs05/24, 202.lssd530-hs05/25, 202.lssd530-hs05/26, 202.lssd530-hs05/27, 202.lssd530-hs05/28, 202.lssd530-hs05/29, 202.lssd530-hs05/30, 202.lssd530-hs05/31, 202.lssd530-hs05/32, 202.lssd530-hs05/33, 202.lssd530-hs05/34, 202.lssd530-hs05/35, 202.lssd530-hs05/36, 202.lssd530-hs05/37, 202.lssd530-hs05/38, 202.lssd530-hs05/39, 203.lssd530-hs05/40, 203.lssd530-hs05/41, 203.lssd530-hs05/42, 203.lssd530-hs05/43, 203.lssd530-hs05/44, 204.lssd530-hs05/45, 204.lssd530-hs05/46, 204.lssd530-hs05/47, 204.lssd530-hs05/48, 204.lssd530-hs05/49, 205.lssd530-hs05/50, 205.lssd530-hs05/51, 205.lssd530-hs05/52, 205.lssd530-hs05/53, 205.lssd530-hs05/54, 206.lssd530-hs05/55, 206.lssd530-hs05/56, 206.lssd530-hs05/57, 206.lssd530-hs05/58, 206.lssd530-hs05/59
     resources_available.arch = linux
     resources_available.host = lssd530-cs12
     resources_available.mem = 395571708kb
     resources_available.ncpus = 104
     resources_available.Qlist = xeon1800w,xeon1800s
     resources_available.vnode = lssd530-cs12
     resources_assigned.accelerator_memory = 0kb
     resources_assigned.hbmem = 0kb
     resources_assigned.mem = 0kb
     resources_assigned.naccelerators = 0
     resources_assigned.ncpus = 60
     resources_assigned.vmem = 0kb
     queue = xeon1800s
     resv_enable = True
     sharing = default_shared
     last_state_change_time = Fri Feb 5 12:29:23 2021
     last_used_time = Fri Feb 5 13:12:49 2021

lssd530-cs13
     Mom = lssd530-cs13
     ntype = PBS
     state = free
     pcpus = 104
     resources_available.arch = linux
     resources_available.host = lssd530-cs13
     resources_available.mem = 395571708kb
     resources_available.ncpus = 104
     resources_available.Qlist = xeon1800w,xeon1800s
     resources_available.vnode = lssd530-cs13
     resources_assigned.accelerator_memory = 0kb
     resources_assigned.hbmem = 0kb
     resources_assigned.mem = 0kb
     resources_assigned.naccelerators = 0
     resources_assigned.ncpus = 0
     resources_assigned.vmem = 0kb
     queue = xeon1800s
     resv_enable = True
     sharing = default_shared
     last_state_change_time = Fri Feb 5 12:29:38 2021
     last_used_time = Fri Feb 5 13:12:49 2021

lssd530-cs09
     Mom = lssd530-cs09
     ntype = PBS
     state = free
     pcpus = 52
     resources_available.arch = linux
     resources_available.host = lssd530-cs09
     resources_available.mem = 65744424kb
     resources_available.ncpus = 104
     resources_available.Qlist = xeon1700w,xeon1700s
     resources_available.vnode = lssd530-cs09
     resources_assigned.accelerator_memory = 0kb
     resources_assigned.hbmem = 0kb
     resources_assigned.mem = 0kb
     resources_assigned.naccelerators = 0
     resources_assigned.ncpus = 0
     resources_assigned.vmem = 0kb
     queue = xeon1700s
     resv_enable = True
     sharing = default_shared
     last_state_change_time = Thu Feb 4 02:52:08 2021
     last_used_time = Thu Feb 4 02:50:14 2021
```