How to stop preemption of the running jobs by high priority jobs

hello,

I am handling a cluster with 40 nodes and in that different normal queues with two high-priority queue has been implemented(queue details given below). now the problem is that whenever a user submits a job using high priority queue(hpq), that hpq job are taking resources from other jobs which are in the running (R) state, if the required resources are not available for that hpq job and because of that other running (R) jobs are going to standBy (S) state. how to prevent that from happening?

queue details:

Create and define queue hpq_1day_large

create queue hpq_1day_large
set queue hpq_1day_large queue_type = Execution
set queue hpq_1day_large Priority = 500
set queue hpq_1day_large max_queued = [o:PBS_ALL=5]
set queue hpq_1day_large acl_user_enable = True
set queue hpq_1day_large acl_users = cdsjsar
set queue hpq_1day_large acl_users += secmon
set queue hpq_1day_large resources_max.nodect = 29
set queue hpq_1day_large resources_max.walltime = 24:00:00
set queue hpq_1day_large resources_min.ncpus = 16
set queue hpq_1day_large resources_min.nodect = 1
set queue hpq_1day_large resources_default.walltime = 24:00:00
set queue hpq_1day_large acl_group_enable = False
set queue hpq_1day_large acl_groups = hpqCPU
set queue hpq_1day_large default_chunk.nodetype = cpunode
set queue hpq_1day_large default_chunk.Qlist = hrl
set queue hpq_1day_large enabled = True
set hpq_gpu_1day started = True

Create and define queue qreg_1day_small

create queue qreg_1day_small
set queue qreg_1day_small queue_type = Execution
set queue qreg_1day_small max_queued = [o:PBS_ALL=100]
set queue qreg_1day_small max_queued += [u:PBS_GENERIC=5]
set queue qreg_1day_small resources_max.ncpus = 127
set queue qreg_1day_small resources_max.nodect = 29
set queue qreg_1day_small resources_max.walltime = 24:00:00
set queue qreg_1day_small resources_min.ncpus = 1
set queue qreg_1day_small resources_min.nodect = 1
set queue qreg_1day_small resources_default.walltime = 24:00:00
set queue qreg_1day_small default_chunk.nodetype = cpunode
set queue qreg_1day_small default_chunk.Qlist = qrs
set queue qreg_1day_small max_run = [o:PBS_ALL=75]
set queue qreg_1day_small max_run += [u:PBS_GENERIC=5]
set queue qreg_1day_small enabled = True
set queue qreg_1day_small started = True

Create and define queue qssd_1day_small

create queue qssd_1day_small
set queue qssd_1day_small queue_type = Execution
set queue qssd_1day_small max_queued = [o:PBS_ALL=20]
set queue qssd_1day_small max_queued += [u:PBS_GENERIC=2]
set queue qssd_1day_small resources_max.ncpus = 127
set queue qssd_1day_small resources_max.nodect = 8
set queue qssd_1day_small resources_max.walltime = 24:00:00
set queue qssd_1day_small resources_min.ncpus = 1
set queue qssd_1day_small resources_min.nodect = 1
set queue qssd_1day_small resources_default.walltime = 24:00:00
set queue qssd_1day_small default_chunk.nodetype = cpunode
set queue qssd_1day_small default_chunk.Qlist = qss
set queue qssd_1day_small max_run = [o:PBS_ALL=10]
set queue qssd_1day_small max_run += [u:PBS_GENERIC=1]
set queue qssd_1day_small enabled = True
set queue qssd_1day_small started = True

Create and define queue qreg_1day_large

create queue qreg_1day_large
set queue qreg_1day_large queue_type = Execution
set queue qreg_1day_large max_queued = [o:PBS_ALL=20]
set queue qreg_1day_large max_queued += [u:PBS_GENERIC=2]
set queue qreg_1day_large resources_max.ncpus = 512
set queue qreg_1day_large resources_max.nodect = 29
set queue qreg_1day_large resources_max.walltime = 24:00:00
set queue qreg_1day_large resources_min.ncpus = 256
set queue qreg_1day_large resources_min.nodect = 1
set queue qreg_1day_large resources_default.walltime = 24:00:00
set queue qreg_1day_large default_chunk.nodetype = cpunode
set queue qreg_1day_large default_chunk.Qlist = qrl
set queue qreg_1day_large max_run = [o:PBS_ALL=10]
set queue qreg_1day_large max_run += [u:PBS_GENERIC=1]
set queue qreg_1day_large enabled = True
set queue qreg_1day_large started = True

and others.

Please follow these steps as root user on the PBS Server:

  1. source /etc/pbs.conf
  2. cd $PBS_HOME/sched_priv
  3. cp sched_config sched_config.bck
  4. edit sched_config and find and update this line to
    preemptive_sched: false ALL
  5. save and exit
  6. kill -HUP # ps -ef | grep pbs_ , get the PID of the pbs_sched

Dear Adarsh,

Thank You for your valuable and informative response.

Regards

Mafizul

1 Like

Dear Adarsh,

I have followed the steps as you have mentioned above.
Now hpq jobs are waiting at the queue when the required resources are not available.
But now the problem is that hpq jobs are at waiting at the queue for their required resources but it is not letting other non-hpq jobs to bypass the queue even though resources are available non-hpq jobs as they require less resources compared to hpq job.

[root@rnarasimha ~]# fn
node1 free 2 2 0 227gb/377gb 32/48 0/0 0/0 69714,69715
node5 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node6 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node7 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node10 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node11 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node13 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node14 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node15 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node16 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node17 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node19 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node20 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
node21 free 0 0 0 377gb/377gb 48/48 0/0 0/0 –
ssdnode6 free 1 1 0 277gb/377gb 8/48 0/0 0/0 69657
ssdnode7 free 1 1 0 257gb/377gb 8/48 0/0 0/0 69697
gpunode1 free 1 1 0 376gb/376gb 16/48 0/0 4/4 69688
gpunode2 free 0 0 0 376gb/376gb 48/48 0/0 4/4 –

[root@rnarasimha ~]# qq
69591.rnarasimha ompijob cdsjsar 0 Q hpq_1day_large
69593.rnarasimha ompijob cdsjsar 0 Q hpq_1day_large
69600.rnarasimha ompijob cdsjsar 0 Q hpq_1day_large
69638.rnarasimha na4 metahma 0 Q qreg_3day_small
69669.rnarasimha 2_SrCeFeO3_d methas 0 Q qreg_3day_small
69682.rnarasimha freq ochsin 0 Q qreg_1day_small
69685.rnarasimha re0.8_512_st chearka 0 Q qreg_1day_small
69686.rnarasimha re1.2_512_st chearka 0 Q qreg_1day_small
69690.rnarasimha s_freq ochsin 0 Q qreg_1day_small
69691.rnarasimha fe_oh ochreena 0 Q qreg_1day_small
69692.rnarasimha fe_oh1 ochreena 0 Q qreg_1day_small
69693.rnarasimha fe_oh2 ochreena 0 Q qreg_1day_small
69694.rnarasimha nuc_attack ochsin 0 Q qreg_1day_small
69695.rnarasimha oxo_attack ochsin 0 Q qreg_1day_small
69700.rnarasimha fe_oh3 ochreena 0 Q qreg_1day_small
69701.rnarasimha test sscbhatt 0 Q qreg_1day_small
69704.rnarasimha 2_SrCeCrO3_d metsachi 0 Q qreg_3day_small
69705.rnarasimha 8_SrCeCrO3_d metsachi 0 Q qreg_3day_small
69709.rnarasimha S03-C1 ochsara 0 Q qssd_1day_small
69710.rnarasimha v2o5_1200_4 metvij 0 Q qreg_1day_small
69711.rnarasimha v2o5_300_6 metvij 0 Q qreg_1day_small
69712.rnarasimha 11_v2o5_600 metvij 0 Q qreg_1day_small
69713.rnarasimha 5_v2o5_900 metvij 0 Q qreg_1day_small
69716.rnarasimha S10 ochsara 0 Q qreg_1day_small
69718.rnarasimha testjob ipcmanoj 0 Q qssd_1day_small
69719.rnarasimha test_script3.sh secmon 0 Q qreg_1day_small
69720.rnarasimha rh2_Xant_Opiv_n ochrogog 0 Q qssd_1day_small
[root@rnarasimha ~]#

Is there any way to bypass non-hpq jobs when the resources are available for them?

Now hpq jobs are waiting at the queue when the required resources are not available.
####Yes, the preemption has been disabled

But now the problem is that hpq jobs are at waiting at the queue for their required resources but it is not letting other non-hpq jobs to bypass the queue even though resources are available non-hpq jobs as they require less resources compared to hpq job.
####You would need to check the scheduler logs (increase the verbosity for one scheduling cycle and set it back to default log level) and scheduling policy to find out why

Also you can check the output of qstat -answ1