In one of oracle application server , the run queue utilization gets abruptly high however at same time CPU utilization remains remarkably low (on an average its Just 50% utilized (upmost) and any corresponding spike on CPU utilization is not observed.)

As result,when some additional traffic diverted to the server, renders it ill responsive.

I/0 stats are just fine.

Can anyone please help me to understand the dynamics behind this phenomenon and kindly suggest a work around to deal with the situation.

[email]root@apps: /root ># vmstat 2 20
procs -----------memory---------- ---swap-- -----io---- --system-- ----cpu----
r b swpd free buff cache si so bi bo in cs us sy id wa
2 0 0 43170184 49648 3098692 0 0 2 15 0 3 12 3 85 0
5 0 0 43167976 49696 3098904 0 0 10 216 5090 24905 25 9 66 0
8 0 0 43257640 49768 3099872 0 0 24 550 5456 24839 18 9 73 0
1 0 0 43274968 49824 3099556 0 0 24 132 7499 27877 22 8 69 0
47 0 0 43257128 49900 3099220 0 0 12 1058 7056 28306 36 11 52 0
12 0 0 43234664 49944 3099696 0 0 12 8 7206 28147 32 12 57 0
2 0 0 43248552 49992 3099908 0 0 2 1150 2915 22616 28 14 58 0
760 0 0 43290392 50020 3100140 0 0 6 144 4740 25029 29 9 62 0
2 0 0 43323800 50036 3100384 0 0 8 0 4159 23350 24 11 65 0
3 0 0 43356568 50100 3101100 0 0 0 840 5054 24218 22 14 65 0
28 0 0 43372448 50124 3100816 0 0 18 28 3771 21515 17 7 76 0
0 0 0 43379488 50148 3100792 0 0 6 380 2705 20216 9 10 81 0
2 0 0 43388960 50236 3101744 0 0 20 326 3506 22740 26 8 65 0
62 0 0 43386592 50236 3102004 0 0 0 0 4880 24178 16 12 72 0
8 1 0 43391456 50312 3101928 0 0 24 564 5071 24717 28 9 63 0
82 0 0 43393184 50452 3102048 0 0 54 388 5934 26600 28 9 62 1
1 0 0 43409440 50504 3101996 0 0 16 684 4821 24301 22 9 69 0
6 0 0 43429480 50556 3102204 0 0 0 708 4169 23673 23 9 67 0
1 0 0 43428312 50596 3102684 0 0 10 30 5212 24842 23 13 65 0
29 0 0 43460424 50652 3102628 0 0 8 642 4553 24106 23 10 67 0

root@Apps: /root ># grep processor /proc/cpuinfo
processor : 0
processor : 1
processor : 2
processor : 3
processor : 4
processor : 5
processor : 6
processor : 7
processor : 8
processor : 9
processor : 10
processor : 11
processor : 12
processor : 13
processor : 14
processor : 15
root@apps: /root ># iostat 2 3
Linux 2.6.9-89.ELsmp (Apps) 08/17/2013

avg-cpu: %user %nice %sys %iowait %idle
11.57 0.00 3.24 0.08 85.11

Device: tps Blk_read/s Blk_wrtn/s Blk_read Blk_wrtn
vda 24.59 72.73 474.90 60249762 393406668
vda1 0.00 0.00 0.00 1374 4
vda2 0.00 0.00 0.00 1012 0
vda3 6.63 10.25 118.54 8494940 98202312
vda5 2.99 9.66 69.60 8001538 57653824
vda6 14.97 52.81 286.76 43750466 237550528
dm-0 32.36 30.43 244.39 25208602 202450440
dm-1 4.00 4.85 27.19 4015146 22527312
dm-2 0.05 0.28 0.16 231514 129168
dm-3 1.91 13.32 6.63 11031106 5494264
dm-4 1.10 3.94 8.39 3263234 6949344
dm-5 15.88 9.75 117.28 8074386 97157184
dm-6 0.22 0.51 1.26 419746 1045128

avg-cpu: %user %nice %sys %iowait %idle
26.00 0.03 11.50 0.13 62.34

Device: tps Blk_read/s Blk_wrtn/s Blk_read Blk_wrtn
vda 44.39 24.49 1375.51 48 2696
vda1 0.00 0.00 0.00 0 0
vda2 0.00 0.00 0.00 0 0
vda3 0.00 0.00 0.00 0 0
vda5 1.02 0.00 69.39 0 136
vda6 43.37 24.49 1306.12 48 2560
dm-0 134.69 24.49 1053.06 48 2064
dm-1 21.43 0.00 171.43 0 336
dm-2 0.00 0.00 0.00 0 0
dm-3 4.59 0.00 36.73 0 72
dm-4 5.61 0.00 44.90 0 88
dm-5 0.00 0.00 0.00 0 0
dm-6 0.00 0.00 0.00 0 0

avg-cpu: %user %nice %sys %iowait %idle
27.61 0.00 10.04 0.10 62.25

Device: tps Blk_read/s Blk_wrtn/s Blk_read Blk_wrtn
vda 14.14 20.20 234.34 40 464
vda1 0.00 0.00 0.00 0 0
vda2 0.00 0.00 0.00 0 0
vda3 11.62 4.04 226.26 8 448
vda5 0.00 0.00 0.00 0 0
vda6 2.53 16.16 8.08 32 16
dm-0 3.03 16.16 8.08 32 16
dm-1 0.00 0.00 0.00 0 0
dm-2 0.00 0.00 0.00 0 0
dm-3 0.00 0.00 0.00 0 0
dm-4 0.00 0.00 0.00 0 0
dm-5 28.79 4.04 226.26 8 448
dm-6 0.00 0.00 0.00 0 0


top - 11:31:09 up 9 days, 16:27, 6 users, load average: 45.36, 94.98, 115.12
Tasks: 1206 total, 3 running, 1203 sleeping, 0 stopped, 0 zombie
Cpu(s): 21.8% us, 7.2% sy, 0.0% ni, 70.5% id, 0.1% wa, 0.4% hi, 0.0% si
Mem: 66052404k total, 22185988k used, 43866416k free, 54932k buffers
Swap: 33551744k total, 0k used, 33551744k free, 3120968k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
6045 ias 16 0 601m 95m 27m S 33.1 0.1 0:18.35 java
31036 ias 16 0 582m 76m 27m S 19.5 0.1 0:17.20 java
8565 ias 15 0 375m 46m 22m S 18.5 0.1 0:57.85 frmweb
7317 ias 17 0 67944 22m 12m S 16.5 0.0 0:01.39 frmweb
7549 ias 15 0 65480 21m 11m S 15.5 0.0 0:00.73 frmweb
4297 ias 16 0 590m 83m 27m S 13.9 0.1 0:17.53 java
7339 ias 16 0 60564 17m 10m S 8.6 0.0 0:00.32 frmweb
7591 ias 16 0 60704 17m 10m S 8.3 0.0 0:00.31 frmweb
5044 ias 17 0 1013m 223m 25m S 5.6 0.3 55:10.36 java
7608 ias 17 0 60420 16m 10m R 5.3 0.0 0:00.22 frmweb
24752 ias 16 0 63440 19m 11m S 4.3 0.0 0:00.63 frmweb
20939 ias 16 0 1016m 219m 25m S 3.6 0.3 231:48.50 java
20943 ias 16 0 881m 513m 40m S 3.6 0.8 204:41.29 java
7797 ias 16 0 58556 15m 10m S 3.0 0.0 0:00.09 frmweb
11790 ias 16 0 1013m 222m 25m S 3.0 0.3 68:37.95 java
23318 ias 17 0 532m 260m 30m S 2.6 0.4 91:06.71 java
14065 ias 15 0 374m 46m 22m S 1.7 0.1 0:16.25 frmweb
15707 ias 15 0 733m 113m 21m S 1.7 0.2 19:32.02 rwserver
26196 ias 15 0 382m 50m 23m S 1.7 0.1 0:07.15 frmweb
1582 ias 15 0 373m 44m 22m S 1.3 0.1 0:44.76 frmweb
1898 ias 15 0 379m 49m 23m S 1.3 0.1 0:47.14 frmweb
7312 ias 15 0 81792 13m 4268 S 1.3 0.0 0:00.18 httpd
22949 ias 15 0 376m 47m 22m S 1.3 0.1 0:45.50 frmweb
1458 ias 16 0 373m 43m 22m S 1.0 0.1 0:04.65 frmweb
1765 ias 15 0 376m 46m 23m S 1.0 0.1 0:04.39 frmweb
5050 ias 15 0 375m 46m 22m S 1.0 0.1 0:58.18 frmweb
6915 ias 16 0 372m 43m 22m S 1.0 0.1 0:01.83 frmweb
7266 ias 15 0 372m 43m 22m S 1.0 0.1 0:01.59 frmweb
7788 root 16 0 2936 1696 788 R 1.0 0.0 0:00.06 top
12247 ias 15 0 81792 14m 5000 S 1.0 0.0 1:41.16 httpd
12300 ias 15 0 82816 14m 5016 S 1.0 0.0 1:43.20 httpd
13301 ias 16 0 372m 43m 22m S 1.0 0.1 0:13.43 frmweb
13854 ias 16 0 378m 49m 23m S 1.0 0.1 0:15.51 frmweb
14874 ias 16 0 372m 43m 22m S 1.0 0.1 0:12.19 frmweb
21339 ias 16 0 372m 43m 22m S 1.0 0.1 0:09.89 frmweb
22533 ias 17 0 3128 1800 780 S 1.0 0.0 9:43.39 top
22614 ias 16 0 371m 41m 21m S 1.0 0.1 0:45.61 frmweb
26687 ias 16 0 386m 54m 24m S 1.0 0.1 0:10.60 frmweb
27267 ias 16 0 384m 52m 24m S 1.0 0.1 0:46.35 frmweb
28053 ias 16 0 372m 43m 22m S 1.0 0.1 0:06.65 frmweb
28786 ias 16 0 374m 45m 23m S 1.0 0.1 0:30.14 frmweb
28858 ias 15 0 370m 42m 21m S 1.0 0.1 0:06.28 frmweb
30055 ias 16 0 376m 45m 22m S 1.0 0.1 0:18.77 frmweb
31614 ias 16 0 379m 50m 22m S 1.0 0.1 1:05.16 frmweb
32392 ias 15 0 375m 45m 22m S 1.0 0.1 0:02.91 frmweb
32417 ias 15 0 379m 48m 23m S 1.0 0.1 0:06.73 frmweb