PostgreSQL 9.2 benchmarks

Francois Tigeot ftigeot at wolfpond.org
Wed Aug 29 05:15:21 PDT 2012


--vkogqOf2sHV7VnPd
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline

On Tue, Aug 28, 2012 at 08:51:43PM +0100, Mindaugas Rasiukevicius wrote:
> 
> If the kernel is before 15th of August, netbsd-6 still had DIAGNOSTIC
> option enabled by default, which would affect the performance.  Did you
> use a kernel with the option disabled?

I'm not sure, I used a snapshot iso from the 16th and uname -a only reports
NetBSD 6.0_BETA2

> Having lockstat output during the benchmark with 24 clients (peak) and
> separately with ~32 (when the system gets barely responsive) could help
> us to understand the problem better.

I couldn't get lockstat results for a complete 10 minutes run, I had to
reduce the time allocated to pgbench to 60 seconds:

This time, the system started feeling unresponsive after the first
24-clients pgbench run.

Results are in the attached files.

-- 
Francois Tigeot

--vkogqOf2sHV7VnPd
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="lockstat_24.txt"

transaction type: SELECT only
scaling factor: 800
query mode: simple
number of clients: 24
number of threads: 24
duration: 60 s
number of transactions actually processed: 582276
tps = 9027.509551 (including connections establishing)
tps = 9040.981235 (excluding connections establishing)
Elapsed time: 64.54 seconds.

-- Adaptive mutex spin

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 90.69 7397761  86424.40 fffffe81301694c0       <all>
 84.74 7397748  80756.67 fffffe81301694c0       uvm_fault_internal+62e
  5.95      12   5667.69 fffffe81301694c0       uvm_unmap_remove+245
  0.00       1      0.03 fffffe81301694c0       uao_reference+1a
  7.14 1080226   6800.32 semlock                <all>
  6.15  872235   5862.53 semlock                cv_wait_sig+10d
  0.98  207991    937.79 semlock                sys_semop+d0
  1.39    3486   1326.80 fffffe81301696c0       <all>
  0.79    3429    751.02 fffffe81301696c0       uvm_fault_internal+62e
  0.60      57    575.78 fffffe81301696c0       uvm_unmap_remove+245
  0.30   99375    288.45 fffffe8635a124c0       <all>
  0.14   39973    132.77 fffffe8635a124c0       uvm_fault_internal+5c0
  0.09   30159     82.20 fffffe8635a124c0       ubc_release+ab
  0.08   29243     73.48 fffffe8635a124c0       ubc_alloc+74
  0.27      39    253.69 fffffe8130169c00       <all>
  0.27      25    253.64 fffffe8130169c00       uvm_map_lock_entry+1d
  0.00      11      0.03 fffffe8130169c00       uvm_fault_internal+341
  0.00       2      0.01 fffffe8130169c00       amap_ref+1f
  0.00       1      0.00 fffffe8130169c00       amap_copy+ab
  0.07    4306     68.33 fffffe812dc62100       <all>
  0.07    4256     66.97 fffffe812dc62100       uvm_fault_internal+341
  0.00      27      0.82 fffffe812dc62100       amap_copy+ab
  0.00      23      0.55 fffffe812dc62100       amap_ref+1f
  0.06  139857     52.42 uvm_pageqlock          <all>
  0.03   71872     26.79 uvm_pageqlock          uvm_fault_lower_enter+243
  0.02   52296     18.73 uvm_pageqlock          ubc_fault+303
  0.00    8036      2.92 uvm_pageqlock          genfs_getpages+8d3
  0.00    5008      2.40 uvm_pageqlock          uvm_fault_internal+e5f
  0.00    1103      0.67 uvm_pageqlock          uvm_fault_internal+4c7
  0.00     955      0.58 uvm_pageqlock          uvm_fault_upper_enter.clone.4+1
  0.00     575      0.32 uvm_pageqlock          uvm_fault_internal+1160
  0.00      12      0.00 uvm_pageqlock          uvm_anon_dispose+11c
  0.01    2716      7.73 fffffe82c0fb0000       <all>
  0.00     911      2.39 fffffe82c0fb0000       vn_lock+c9
  0.00     657      1.74 fffffe82c0fb0000       ubc_alloc+5a8
  0.00     413      1.65 fffffe82c0fb0000       genfs_getpages+396
  0.00     534      1.30 fffffe82c0fb0000       ubc_fault+14b
  0.00     201      0.65 fffffe82c0fb0000       ubc_fault+1e2
  0.01    2797      7.51 fffffe82c1a69e80       <all>
  0.00     888      2.25 fffffe82c1a69e80       vn_lock+c9
  0.00     503      1.96 fffffe82c1a69e80       genfs_getpages+396
  0.00     676      1.72 fffffe82c1a69e80       ubc_alloc+5a8
  0.00     511      0.91 fffffe82c1a69e80       ubc_fault+14b
  0.00     218      0.66 fffffe82c1a69e80       ubc_fault+1e2
  0.00       1      0.00 fffffe82c1a69e80       vn_close+58
  0.01    2630      7.36 fffffe82c1a69f00       <all>
  0.00     876      2.28 fffffe82c1a69f00       vn_lock+c9
  0.00     705      1.88 fffffe82c1a69f00       ubc_alloc+5a8
  0.00     375      1.44 fffffe82c1a69f00       genfs_getpages+396
  0.00     481      1.11 fffffe82c1a69f00       ubc_fault+14b
  0.00     193      0.65 fffffe82c1a69f00       ubc_fault+1e2
  0.01    2548      7.10 fffffe82c1a69f80       <all>
  0.00     909      2.36 fffffe82c1a69f80       vn_lock+c9
  0.00     629      1.69 fffffe82c1a69f80       ubc_alloc+5a8
  0.00     358      1.39 fffffe82c1a69f80       genfs_getpages+396
  0.00     476      1.21 fffffe82c1a69f80       ubc_fault+14b
  0.00     176      0.47 fffffe82c1a69f80       ubc_fault+1e2
  0.01    2599      7.08 fffffe82c0fb0100       <all>
  0.00     854      2.16 fffffe82c0fb0100       vn_lock+c9
  0.00     623      1.62 fffffe82c0fb0100       ubc_alloc+5a8
  0.00     383      1.49 fffffe82c0fb0100       genfs_getpages+396
  0.00     558      1.19 fffffe82c0fb0100       ubc_fault+14b
  0.00     181      0.63 fffffe82c0fb0100       ubc_fault+1e2
  0.01    2598      7.06 fffffe82c0fb0080       <all>
  0.00     867      2.23 fffffe82c0fb0080       vn_lock+c9
  0.00     633      1.55 fffffe82c0fb0080       ubc_alloc+5a8
  0.00     404      1.54 fffffe82c0fb0080       genfs_getpages+396
  0.00     516      1.18 fffffe82c0fb0080       ubc_fault+14b
  0.00     178      0.57 fffffe82c0fb0080       ubc_fault+1e2
  0.01    2563      7.04 fffffe82c0fb00c0       <all>
  0.00     891      2.19 fffffe82c0fb00c0       vn_lock+c9
  0.00     617      1.72 fffffe82c0fb00c0       ubc_alloc+5a8
  0.00     393      1.52 fffffe82c0fb00c0       genfs_getpages+396
  0.00     488      1.05 fffffe82c0fb00c0       ubc_fault+14b
  0.00     174      0.57 fffffe82c0fb00c0       ubc_fault+1e2
  0.01    2539      7.01 fffffe82c0fb0040       <all>
  0.00     867      2.32 fffffe82c0fb0040       vn_lock+c9
  0.00     651      1.63 fffffe82c0fb0040       ubc_alloc+5a8
  0.00     353      1.36 fffffe82c0fb0040       genfs_getpages+396
  0.00     486      1.17 fffffe82c0fb0040       ubc_fault+14b
  0.00     182      0.53 fffffe82c0fb0040       ubc_fault+1e2
  0.01    2521      6.99 fffffe82c1a69f40       <all>
  0.00     846      2.17 fffffe82c1a69f40       vn_lock+c9
  0.00     622      1.57 fffffe82c1a69f40       ubc_alloc+5a8
  0.00     370      1.50 fffffe82c1a69f40       genfs_getpages+396
  0.00     500      1.18 fffffe82c1a69f40       ubc_fault+14b
  0.00     183      0.56 fffffe82c1a69f40       ubc_fault+1e2
  0.01    2511      6.93 fffffe82c1a69ec0       <all>
  0.00     831      2.18 fffffe82c1a69ec0       vn_lock+c9
  0.00     631      1.61 fffffe82c1a69ec0       ubc_alloc+5a8
  0.00     366      1.42 fffffe82c1a69ec0       genfs_getpages+396
  0.00     496      1.08 fffffe82c1a69ec0       ubc_fault+14b
  0.00     186      0.63 fffffe82c1a69ec0       ubc_fault+1e2
  0.00       1      0.00 fffffe82c1a69ec0       vn_close+58
  0.00     269      3.32 fffffe84f0438a80       uvm_fault_internal+62e
  0.00     804      1.86 fffffe863f3acf40       <all>
  0.00     193      0.50 fffffe863f3acf40       soreceive+db
  0.00     170      0.47 fffffe863f3acf40       sosend+4c
  0.00     149      0.42 fffffe863f3acf40       sopoll+1c
  0.00     108      0.21 fffffe863f3acf40       sosend+2cd
  0.00      56      0.11 fffffe863f3acf40       soreceive+86c
  0.00     104      0.11 fffffe863f3acf40       ip6intr+13
  0.00      24      0.06 fffffe863f3acf40       cv_wait_sig+10d
  0.00      40      1.57 fffffe812ebc8bc0       <all>
  0.00      39      1.57 fffffe812ebc8bc0       uvm_fault_internal+341
  0.00       1      0.01 fffffe812ebc8bc0       amap_ref+1f
  0.00     228      1.09 xc_low_pri             <all>
  0.00     163      0.70 xc_low_pri             cv_wait+109
  0.00      51      0.31 xc_low_pri             xc_unicast+75
  0.00       9      0.07 xc_low_pri             xc_thread+88
  0.00       5      0.01 xc_low_pri             xc_wait+68
  0.00      34      0.63 fffffe8130169980       <all>
  0.00      32      0.61 fffffe8130169980       uvm_fault_internal+341
  0.00       2      0.02 fffffe8130169980       amap_copy+ab
  0.00     292      0.32 fffffe82c1a69cc0       <all>
  0.00     287      0.31 fffffe82c1a69cc0       vn_lock+c9
  0.00       3      0.00 fffffe82c1a69cc0       genfs_getpages+396
  0.00       2      0.00 fffffe82c1a69cc0       ubc_fault+1e2
  0.00     274      0.24 fffffe82c1a69e40       <all>
  0.00     272      0.23 fffffe82c1a69e40       vn_lock+c9
  0.00       1      0.00 fffffe82c1a69e40       ubc_fault+14b
  0.00       1      0.00 fffffe82c1a69e40       genfs_getpages+396
  0.00      32      0.18 fffffe812c98b400       <all>
  0.00      25      0.15 fffffe812c98b400       uvm_fault_internal+341
  0.00       5      0.02 fffffe812c98b400       amap_copy+ab
  0.00       2      0.01 fffffe812c98b400       amap_ref+1f
  0.00      14      0.15 fffffe8130169a40       <all>
  0.00       7      0.11 fffffe8130169a40       uvm_fault_internal+341
  0.00       4      0.02 fffffe8130169a40       amap_copy+ab
  0.00       3      0.02 fffffe8130169a40       amap_ref+1f
  0.00      12      0.08 fffffe8130169b80       <all>
  0.00      11      0.08 fffffe8130169b80       uvm_fault_internal+341
  0.00       1      0.00 fffffe8130169b80       amap_copy+ab
  0.00      11      0.08 fffffe8130169cc0       <all>
  0.00       8      0.06 fffffe8130169cc0       uvm_fault_internal+341
  0.00       2      0.03 fffffe8130169cc0       amap_ref+1f
  0.00       1      0.00 fffffe8130169cc0       amap_copy+ab
  0.00      37      0.05 fffffe85aee69a80       pmap_map_ptes+49
  0.00       2      0.02 fffffe8130169d40       uvm_fault_internal+341
  0.00      34      0.02 fffffe812d641e80       <all>
  0.00      18      0.01 fffffe812d641e80       sopoll+1c
  0.00       7      0.00 fffffe812d641e80       soreceive+86c
  0.00       3      0.00 fffffe812d641e80       cv_timedwait_sig+116
  0.00       2      0.00 fffffe812d641e80       sosend+2cd
  0.00       3      0.00 fffffe812d641e80       soreceive+db
  0.00       1      0.00 fffffe812d641e80       sosend+4c
  0.00      32      0.02 fffffe85b1b42180       <all>
  0.00      16      0.01 fffffe85b1b42180       sopoll+1c
  0.00       4      0.01 fffffe85b1b42180       cv_timedwait_sig+116
  0.00       5      0.00 fffffe85b1b42180       soreceive+86c
  0.00       1      0.00 fffffe85b1b42180       sosend+2cd
  0.00       5      0.00 fffffe85b1b42180       soreceive+db
  0.00       1      0.00 fffffe85b1b42180       sosend+4c
  0.00      45      0.02 fffffe85b0184f40       <all>
  0.00      20      0.01 fffffe85b0184f40       sopoll+1c
  0.00      10      0.00 fffffe85b0184f40       soreceive+db
  0.00       7      0.00 fffffe85b0184f40       soreceive+86c
  0.00       5      0.00 fffffe85b0184f40       cv_timedwait_sig+116
  0.00       3      0.00 fffffe85b0184f40       sosend+2cd
  0.00      30      0.02 fffffe85af0b0d80       <all>
  0.00      17      0.01 fffffe85af0b0d80       sopoll+1c
  0.00       4      0.00 fffffe85af0b0d80       cv_timedwait_sig+116
  0.00       6      0.00 fffffe85af0b0d80       soreceive+86c
  0.00       2      0.00 fffffe85af0b0d80       soreceive+db
  0.00       1      0.00 fffffe85af0b0d80       sosend+4c
  0.00      22      0.02 fffffe85b1b42480       <all>
  0.00       4      0.01 fffffe85b1b42480       cv_timedwait_sig+116
  0.00      10      0.00 fffffe85b1b42480       sopoll+1c
  0.00       4      0.00 fffffe85b1b42480       soreceive+86c
  0.00       2      0.00 fffffe85b1b42480       soreceive+db
  0.00       2      0.00 fffffe85b1b42480       sosend+4c
  0.00      30      0.02 fffffe812cde2e40       <all>
  0.00       9      0.01 fffffe812cde2e40       soreceive+db
  0.00       9      0.00 fffffe812cde2e40       sopoll+1c
  0.00       6      0.00 fffffe812cde2e40       cv_timedwait_sig+116
  0.00       4      0.00 fffffe812cde2e40       soreceive+86c
  0.00       2      0.00 fffffe812cde2e40       sosend+2cd
  0.00      34      0.02 fffffe82c1a69a00       <all>
  0.00       4      0.01 fffffe82c1a69a00       cv_timedwait_sig+116
  0.00       7      0.00 fffffe82c1a69a00       soreceive+86c
  0.00      14      0.00 fffffe82c1a69a00       sopoll+1c
  0.00       7      0.00 fffffe82c1a69a00       soreceive+db
  0.00       1      0.00 fffffe82c1a69a00       sosend+2cd
  0.00       1      0.00 fffffe82c1a69a00       sosend+4c
  0.00      35      0.02 fffffe8130169d80       <all>
  0.00       8      0.00 fffffe8130169d80       cv_timedwait_sig+116
  0.00      14      0.00 fffffe8130169d80       sopoll+1c
  0.00       6      0.00 fffffe8130169d80       soreceive+db
  0.00       2      0.00 fffffe8130169d80       sosend+2cd
  0.00       3      0.00 fffffe8130169d80       soreceive+86c
  0.00       2      0.00 fffffe8130169d80       sosend+4c
  0.00      24      0.02 fffffe862252eec0       <all>
  0.00      10      0.01 fffffe862252eec0       sopoll+1c
  0.00       6      0.00 fffffe862252eec0       cv_timedwait_sig+116
  0.00       6      0.00 fffffe862252eec0       soreceive+db
  0.00       2      0.00 fffffe862252eec0       sosend+2cd
  0.00      29      0.01 fffffe812c0b4dc0       <all>
  0.00      12      0.01 fffffe812c0b4dc0       sopoll+1c
  0.00       5      0.00 fffffe812c0b4dc0       soreceive+86c
  0.00       5      0.00 fffffe812c0b4dc0       cv_timedwait_sig+116
  0.00       1      0.00 fffffe812c0b4dc0       sosend+2cd
  0.00       5      0.00 fffffe812c0b4dc0       soreceive+db
  0.00       1      0.00 fffffe812c0b4dc0       sosend+4c
  0.00       2      0.01 fffffe812df27d80       amap_copy+ab
  0.00      35      0.01 fffffe812c150540       <all>
  0.00       4      0.00 fffffe812c150540       soreceive+86c
  0.00       6      0.00 fffffe812c150540       soreceive+db
  0.00       9      0.00 fffffe812c150540       sopoll+1c
  0.00      13      0.00 fffffe812c150540       cv_timedwait_sig+116
  0.00       1      0.00 fffffe812c150540       sosend+4c
  0.00       1      0.00 fffffe812c150540       do_sys_getsockname+5f
  0.00       1      0.00 fffffe812c150540       sosend+2cd
  0.00      32      0.01 fffffe812c0b4e80       <all>
  0.00      13      0.01 fffffe812c0b4e80       sopoll+1c
  0.00       6      0.00 fffffe812c0b4e80       soreceive+86c
  0.00       6      0.00 fffffe812c0b4e80       soreceive+db
  0.00       1      0.00 fffffe812c0b4e80       sosend+2cd
  0.00       5      0.00 fffffe812c0b4e80       cv_timedwait_sig+116
  0.00       1      0.00 fffffe812c0b4e80       sosend+4c
  0.00      29      0.01 fffffe85b1b42300       <all>
  0.00       9      0.01 fffffe85b1b42300       soreceive+db
  0.00      10      0.00 fffffe85b1b42300       sopoll+1c
  0.00       6      0.00 fffffe85b1b42300       cv_timedwait_sig+116
  0.00       3      0.00 fffffe85b1b42300       soreceive+86c
  0.00       1      0.00 fffffe85b1b42300       sosend+2cd
  0.00      33      0.01 fffffe862252e740       <all>
  0.00      11      0.00 fffffe862252e740       soreceive+db
  0.00       9      0.00 fffffe862252e740       sopoll+1c
  0.00       5      0.00 fffffe862252e740       soreceive+86c
  0.00       3      0.00 fffffe862252e740       sosend+4c
  0.00       4      0.00 fffffe862252e740       cv_timedwait_sig+116
  0.00       1      0.00 fffffe862252e740       sosend+2cd
  0.00      29      0.01 fffffe82ba910740       <all>
  0.00       9      0.01 fffffe82ba910740       cv_timedwait_sig+116
  0.00      12      0.00 fffffe82ba910740       sopoll+1c
  0.00       4      0.00 fffffe82ba910740       soreceive+db
  0.00       3      0.00 fffffe82ba910740       soreceive+86c
  0.00       1      0.00 fffffe82ba910740       sosend+2cd
  0.00      20      0.01 fffffe85af0b0040       <all>
  0.00       8      0.00 fffffe85af0b0040       sopoll+1c
  0.00       3      0.00 fffffe85af0b0040       soreceive+86c
  0.00       3      0.00 fffffe85af0b0040       soreceive+db
  0.00       3      0.00 fffffe85af0b0040       sosend+2cd
  0.00       2      0.00 fffffe85af0b0040       cv_timedwait_sig+116
  0.00       1      0.00 fffffe85af0b0040       sosend+4c
  0.00      21      0.01 fffffe85af0b0380       <all>
  0.00      10      0.00 fffffe85af0b0380       sopoll+1c
  0.00       5      0.00 fffffe85af0b0380       cv_timedwait_sig+116
  0.00       3      0.00 fffffe85af0b0380       soreceive+86c
  0.00       2      0.00 fffffe85af0b0380       soreceive+db
  0.00       1      0.00 fffffe85af0b0380       sosend+4c
  0.00       7      0.01 fffffe8635a12440       <all>
  0.00       5      0.01 fffffe8635a12440       sopoll+1c
  0.00       1      0.00 fffffe8635a12440       sosend+4c
  0.00       1      0.00 fffffe8635a12440       sosend+2cd
  0.00      20      0.01 fffffe812c98b840       <all>
  0.00      10      0.00 fffffe812c98b840       sopoll+1c
  0.00       3      0.00 fffffe812c98b840       cv_timedwait_sig+116
  0.00       2      0.00 fffffe812c98b840       sosend+2cd
  0.00       4      0.00 fffffe812c98b840       soreceive+86c
  0.00       1      0.00 fffffe812c98b840       soreceive+db
  0.00       1      0.01 fffffe837379fa80       uvm_fault_internal+62e
  0.00      26      0.01 fffffe86230fbf80       <all>
  0.00      12      0.00 fffffe86230fbf80       sopoll+1c
  0.00       4      0.00 fffffe86230fbf80       soreceive+86c
  0.00       6      0.00 fffffe86230fbf80       cv_timedwait_sig+116
  0.00       3      0.00 fffffe86230fbf80       soreceive+db
  0.00       1      0.00 fffffe86230fbf80       sosend+4c
  0.00      28      0.01 fffffe862252ee00       <all>
  0.00       4      0.00 fffffe862252ee00       soreceive+86c
  0.00       5      0.00 fffffe862252ee00       soreceive+db
  0.00       9      0.00 fffffe862252ee00       sopoll+1c
  0.00       1      0.00 fffffe862252ee00       sosend+4c
  0.00       8      0.00 fffffe862252ee00       cv_timedwait_sig+116
  0.00       1      0.00 fffffe862252ee00       do_sys_getsockname+5f
  0.00      22      0.01 fffffe85af4ca480       <all>
  0.00       9      0.00 fffffe85af4ca480       sopoll+1c
  0.00       3      0.00 fffffe85af4ca480       soreceive+db
  0.00       3      0.00 fffffe85af4ca480       sosend+2cd
  0.00       5      0.00 fffffe85af4ca480       cv_timedwait_sig+116
  0.00       2      0.00 fffffe85af4ca480       soreceive+86c
  0.00      23      0.01 fffffe85b2814c80       <all>
  0.00       6      0.00 fffffe85b2814c80       soreceive+86c
  0.00       6      0.00 fffffe85b2814c80       cv_timedwait_sig+116
  0.00       3      0.00 fffffe85b2814c80       soreceive+db
  0.00       1      0.00 fffffe85b2814c80       sosend+2cd
  0.00       6      0.00 fffffe85b2814c80       sopoll+1c
  0.00       1      0.00 fffffe85b2814c80       sosend+4c
  0.00      18      0.01 fffffe82ba910500       <all>
  0.00       6      0.00 fffffe82ba910500       soreceive+db
  0.00       5      0.00 fffffe82ba910500       sopoll+1c
  0.00       2      0.00 fffffe82ba910500       soreceive+86c
  0.00       3      0.00 fffffe82ba910500       cv_timedwait_sig+116
  0.00       1      0.00 fffffe82ba910500       sosend+2cd
  0.00       1      0.00 fffffe82ba910500       sosend+4c
  0.00      11      0.00 fffffe85b01842c0       <all>
  0.00       5      0.00 fffffe85b01842c0       sopoll+1c
  0.00       5      0.00 fffffe85b01842c0       cv_timedwait_sig+116
  0.00       1      0.00 fffffe85b01842c0       soreceive+db
  0.00       1      0.00 fffffe812f148540       uvm_fault_internal+341
  0.00       1      0.00 fffffe84f0438b40       uvm_fault_internal+62e
  0.00       7      0.00 shm_lock               shmexit+20
  0.00      10      0.00 fffffe8130169280       pipe_poll+1a
  0.00       7      0.00 fffffe8130169340       <all>
  0.00       3      0.00 fffffe8130169340       vn_lock+115
  0.00       2      0.00 fffffe8130169340       cache_lookup+112
  0.00       2      0.00 fffffe8130169340       vn_lock+c9
  0.00       6      0.00 pmap_pv_cache+1a0      pool_cache_get_slow+217
  0.00       1      0.00 fffffe8635b0d460       pool_cache_put_slow+153
  0.00       2      0.00 fffffe812c150ac0       <all>
  0.00       1      0.00 fffffe812c150ac0       sys___sigprocmask14+3e
  0.00       1      0.00 fffffe812c150ac0       lwp_exit+80
  0.00       3      0.00 vnode_free_list_lock   <all>
  0.00       2      0.00 vnode_free_list_lock   vremfree+33
  0.00       1      0.00 vnode_free_list_lock   vrelel+413
  0.00       1      0.00 fffffe85b1b421c0       cv_timedwait_sig+116
  0.00       2      0.00 fffffe81301693c0       <all>
  0.00       1      0.00 fffffe81301693c0       vn_lock+115
  0.00       1      0.00 fffffe81301693c0       vn_lock+c9

-- Adaptive mutex sleep

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 98.69       1    101.91 fffffe81301696c0       uvm_fault_internal+62e
  0.75      68      0.78 fffffe81301694c0       uvm_fault_internal+62e
  0.50       4      0.52 fffffe8635a124c0       <all>
  0.30       2      0.31 fffffe8635a124c0       ubc_release+ab
  0.20       2      0.21 fffffe8635a124c0       ubc_alloc+74
  0.05       4      0.05 semlock                <all>
  0.04       3      0.04 semlock                cv_wait_sig+10d
  0.01       1      0.01 semlock                sys_semop+d0
  0.00       1      0.00 fffffe863f3acf40       route_intr+26

-- Spin mutex spin

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 69.99  452072    707.48 fffffe863e341700       <all>
 56.87  363224    574.85 fffffe863e341700       cv_wait_sig+99
 13.12   88833    132.59 fffffe863e341700       cv_wakeup_all+45
  0.00      15      0.03 fffffe863e341700       sched_pstats+c0
 25.04   21148    253.12 uvm_fpageqlock         <all>
 19.96   16242    201.78 uvm_fpageqlock         uvm_pagefree+224
  5.03    4257     50.86 uvm_fpageqlock         uvm_pagealloc_strat+c3
  0.05     633      0.46 uvm_fpageqlock         uvm_pageidlezero+201
  0.00      16      0.02 uvm_fpageqlock         uvm_pglistalloc+b8
  0.31     464      3.10 fffffe863e351800       <all>
  0.31     461      3.09 fffffe863e351800       turnstile_lookup+25
  0.00       3      0.01 fffffe863e351800       turnstile_block+3be
  0.20    6251      2.07 fffffe863e340d00       <all>
  0.15    5152      1.54 fffffe863e340d00       sched_idle+1ad
  0.02     367      0.20 fffffe863e340d00       sleepq_remove+107
  0.02     430      0.18 fffffe863e340d00       sched_idle+123
  0.02     302      0.16 fffffe863e340d00       mi_switch+a9
  0.19    4961      1.95 fffffe8635a12d80       <all>
  0.12    3125      1.21 fffffe8635a12d80       sched_idle+11a
  0.02     818      0.25 fffffe8635a12d80       sched_idle+1ad
  0.02     369      0.21 fffffe8635a12d80       sleepq_remove+107
  0.01     290      0.15 fffffe8635a12d80       mi_switch+a9
  0.01     330      0.12 fffffe8635a12d80       sched_idle+1a3
  0.00      29      0.01 fffffe8635a12d80       sched_idle+123
  0.18    5160      1.86 fffffe863e3407c0       <all>
  0.12    3342      1.19 fffffe863e3407c0       sched_idle+1ad
  0.02     762      0.24 fffffe863e3407c0       sched_idle+11a
  0.02     336      0.17 fffffe863e3407c0       sleepq_remove+107
  0.01     349      0.13 fffffe863e3407c0       mi_switch+a9
  0.01     271      0.09 fffffe863e3407c0       sched_idle+123
  0.00     100      0.03 fffffe863e3407c0       sched_idle+1a3
  0.17    4913      1.72 fffffe863e3405c0       <all>
  0.10    2934      0.98 fffffe863e3405c0       sched_idle+1ad
  0.03    1060      0.30 fffffe863e3405c0       sched_idle+11a
  0.02     324      0.17 fffffe863e3405c0       sleepq_remove+107
  0.01     276      0.14 fffffe863e3405c0       mi_switch+a9
  0.01     196      0.08 fffffe863e3405c0       sched_idle+123
  0.00     122      0.05 fffffe863e3405c0       sched_idle+1a3
  0.00       1      0.00 fffffe863e3405c0       sched_pstats+c0
  0.17    4158      1.70 fffffe8635a12980       <all>
  0.11    2658      1.09 fffffe8635a12980       sched_idle+11a
  0.02     402      0.21 fffffe8635a12980       sleepq_remove+107
  0.01     506      0.13 fffffe8635a12980       sched_idle+1ad
  0.01     264      0.13 fffffe8635a12980       mi_switch+a9
  0.01     306      0.12 fffffe8635a12980       sched_idle+1a3
  0.00      22      0.01 fffffe8635a12980       sched_idle+123
  0.17    4871      1.69 fffffe863e3404c0       <all>
  0.09    2559      0.91 fffffe863e3404c0       sched_idle+1ad
  0.04    1207      0.38 fffffe863e3404c0       sched_idle+11a
  0.02     426      0.21 fffffe863e3404c0       sleepq_remove+107
  0.01     321      0.10 fffffe863e3404c0       mi_switch+a9
  0.01     199      0.05 fffffe863e3404c0       sched_idle+123
  0.00     159      0.04 fffffe863e3404c0       sched_idle+1a3
  0.16    4502      1.62 fffffe863e3406c0       <all>
  0.09    2523      0.92 fffffe863e3406c0       sched_idle+1ad
  0.03    1026      0.34 fffffe863e3406c0       sched_idle+11a
  0.02     348      0.19 fffffe863e3406c0       sleepq_remove+107
  0.01     274      0.09 fffffe863e3406c0       mi_switch+a9
  0.00     187      0.04 fffffe863e3406c0       sched_idle+123
  0.00     144      0.04 fffffe863e3406c0       sched_idle+1a3
  0.16    4868      1.62 fffffe863e340ac0       <all>
  0.11    3758      1.12 fffffe863e340ac0       sched_idle+1ad
  0.02     335      0.17 fffffe863e340ac0       sleepq_remove+107
  0.01     331      0.15 fffffe863e340ac0       mi_switch+a9
  0.01     286      0.10 fffffe863e340ac0       sched_idle+123
  0.01     127      0.08 fffffe863e340ac0       sched_idle+11a
  0.00      31      0.01 fffffe863e340ac0       sched_idle+1a3
  0.16    4085      1.61 fffffe8635a12b80       <all>
  0.09    2323      0.88 fffffe8635a12b80       sched_idle+11a
  0.03     849      0.32 fffffe8635a12b80       sched_idle+1ad
  0.02     356      0.20 fffffe8635a12b80       sleepq_remove+107
  0.01     239      0.11 fffffe8635a12b80       mi_switch+a9
  0.01     270      0.09 fffffe8635a12b80       sched_idle+1a3
  0.00      48      0.01 fffffe8635a12b80       sched_idle+123
  0.16    4852      1.58 fffffe863e3408c0       <all>
  0.12    3394      1.17 fffffe863e3408c0       sched_idle+1ad
  0.02     374      0.17 fffffe863e3408c0       sleepq_remove+107
  0.01     408      0.13 fffffe863e3408c0       sched_idle+11a
  0.01     321      0.05 fffffe863e3408c0       sched_idle+123
  0.00     295      0.05 fffffe863e3408c0       mi_switch+a9
  0.00      60      0.01 fffffe863e3408c0       sched_idle+1a3
  0.15    4364      1.54 fffffe863e3401c0       <all>
  0.06    1635      0.57 fffffe863e3401c0       sched_idle+11a
  0.05    1738      0.54 fffffe863e3401c0       sched_idle+1ad
  0.02     365      0.18 fffffe863e3401c0       sleepq_remove+107
  0.01     293      0.13 fffffe863e3401c0       mi_switch+a9
  0.01     222      0.08 fffffe863e3401c0       sched_idle+1a3
  0.00     111      0.04 fffffe863e3401c0       sched_idle+123
  0.14    4173      1.42 fffffe863e3402c0       <all>
  0.06    1975      0.66 fffffe863e3402c0       sched_idle+1ad
  0.04    1212      0.36 fffffe863e3402c0       sched_idle+11a
  0.02     367      0.18 fffffe863e3402c0       sleepq_remove+107
  0.01     258      0.11 fffffe863e3402c0       mi_switch+a9
  0.01     200      0.07 fffffe863e3402c0       sched_idle+1a3
  0.01     161      0.05 fffffe863e3402c0       sched_idle+123
  0.14    3434      1.38 fffffe8635a12780       <all>
  0.09    2336      0.87 fffffe8635a12780       sched_idle+11a
  0.02     365      0.18 fffffe8635a12780       sleepq_remove+107
  0.01     307      0.13 fffffe8635a12780       sched_idle+1a3
  0.01     218      0.12 fffffe8635a12780       mi_switch+a9
  0.01     195      0.07 fffffe8635a12780       sched_idle+1ad
  0.00      13      0.01 fffffe8635a12780       sched_idle+123
  0.13    3214      1.36 fffffe8635a12e80       <all>
  0.06    1431      0.62 fffffe8635a12e80       sched_idle+11a
  0.03     791      0.28 fffffe8635a12e80       sched_idle+1ad
  0.02     395      0.20 fffffe8635a12e80       sleepq_remove+107
  0.01     294      0.12 fffffe8635a12e80       sched_idle+1a3
  0.01     228      0.11 fffffe8635a12e80       mi_switch+a9
  0.00      75      0.03 fffffe8635a12e80       sched_idle+123
  0.13    3108      1.32 fffffe8635a12c80       <all>
  0.06    1213      0.58 fffffe8635a12c80       sched_idle+11a
  0.04     997      0.36 fffffe8635a12c80       sched_idle+1ad
  0.02     314      0.19 fffffe8635a12c80       sleepq_remove+107
  0.01     227      0.08 fffffe8635a12c80       mi_switch+a9
  0.01     282      0.08 fffffe8635a12c80       sched_idle+1a3
  0.00      75      0.03 fffffe8635a12c80       sched_idle+123
  0.13    3936      1.27 fffffe863e3403c0       <all>
  0.05    1697      0.51 fffffe863e3403c0       sched_idle+1ad
  0.03    1282      0.35 fffffe863e3403c0       sched_idle+11a
  0.02     353      0.17 fffffe863e3403c0       sleepq_remove+107
  0.01     258      0.12 fffffe863e3403c0       mi_switch+a9
  0.01     228      0.08 fffffe863e3403c0       sched_idle+1a3
  0.00     118      0.04 fffffe863e3403c0       sched_idle+123
  0.12    3620      1.21 fffffe8635a12f80       <all>
  0.04    1365      0.42 fffffe8635a12f80       sched_idle+11a
  0.04    1265      0.36 fffffe8635a12f80       sched_idle+1ad
  0.02     363      0.19 fffffe8635a12f80       sleepq_remove+107
  0.01     253      0.12 fffffe8635a12f80       mi_switch+a9
  0.01     275      0.09 fffffe8635a12f80       sched_idle+1a3
  0.00      98      0.04 fffffe8635a12f80       sched_idle+123
  0.00       1      0.00 fffffe8635a12f80       sched_pstats+c0
  0.11    2264      1.13 fffffe8635a12680       <all>
  0.09    1832      0.91 fffffe8635a12680       sched_idle+11a
  0.01     167      0.11 fffffe8635a12680       sleepq_remove+107
  0.01     135      0.06 fffffe8635a12680       mi_switch+a9
  0.01     125      0.05 fffffe8635a12680       sched_idle+1a3
  0.00       5      0.00 fffffe8635a12680       sched_idle+1ad
  0.06    1506      0.59 fffffe8635a12580       <all>
  0.04    1048      0.45 fffffe8635a12580       sched_idle+11a
  0.01     155      0.06 fffffe8635a12580       sleepq_remove+107
  0.00     196      0.04 fffffe8635a12580       sched_idle+1a3
  0.00     107      0.03 fffffe8635a12580       mi_switch+a9
  0.06     147      0.57 pv_hash_locks+5c0      <all>
  0.05     144      0.55 pv_hash_locks+5c0      pmap_remove_pv+b5
  0.00       3      0.02 pv_hash_locks+5c0      pmap_enter_ma+7d9
  0.05     154      0.55 pv_hash_locks+540      <all>
  0.05     152      0.54 pv_hash_locks+540      pmap_remove_pv+b5
  0.00       2      0.01 pv_hash_locks+540      pmap_enter_ma+7d9
  0.05     146      0.54 pv_hash_locks+740      <all>
  0.05     143      0.54 pv_hash_locks+740      pmap_remove_pv+b5
  0.00       3      0.00 pv_hash_locks+740      pmap_enter_ma+7d9
  0.05     143      0.54 pv_hash_locks+280      <all>
  0.05     137      0.50 pv_hash_locks+280      pmap_remove_pv+b5
  0.00       6      0.04 pv_hash_locks+280      pmap_enter_ma+7d9
  0.05     146      0.54 pv_hash_locks+780      <all>
  0.05     141      0.49 pv_hash_locks+780      pmap_remove_pv+b5
  0.00       5      0.05 pv_hash_locks+780      pmap_enter_ma+7d9
  0.05     144      0.53 pv_hash_locks+c0       <all>
  0.05     139      0.51 pv_hash_locks+c0       pmap_remove_pv+b5
  0.00       5      0.02 pv_hash_locks+c0       pmap_enter_ma+7d9
  0.05     148      0.53 pv_hash_locks+240      <all>
  0.05     141      0.49 pv_hash_locks+240      pmap_remove_pv+b5
  0.00       7      0.04 pv_hash_locks+240      pmap_enter_ma+7d9
  0.05     142      0.52 pv_hash_locks+580      <all>
  0.05     135      0.51 pv_hash_locks+580      pmap_remove_pv+b5
  0.00       7      0.01 pv_hash_locks+580      pmap_enter_ma+7d9
  0.05     144      0.52 pv_hash_locks+640      <all>
  0.05     143      0.52 pv_hash_locks+640      pmap_remove_pv+b5
  0.00       1      0.01 pv_hash_locks+640      pmap_enter_ma+7d9
  0.05     143      0.52 pv_hash_locks+6c0      <all>
  0.05     138      0.50 pv_hash_locks+6c0      pmap_remove_pv+b5
  0.00       5      0.02 pv_hash_locks+6c0      pmap_enter_ma+7d9
  0.05     146      0.52 pv_hash_locks+600      <all>
  0.05     143      0.50 pv_hash_locks+600      pmap_remove_pv+b5
  0.00       3      0.02 pv_hash_locks+600      pmap_enter_ma+7d9
  0.05     139      0.51 pv_hash_locks+700      <all>
  0.05     134      0.47 pv_hash_locks+700      pmap_remove_pv+b5
  0.00       5      0.04 pv_hash_locks+700      pmap_enter_ma+7d9
  0.05     142      0.51 pv_hash_locks+400      <all>
  0.05     136      0.48 pv_hash_locks+400      pmap_remove_pv+b5
  0.00       6      0.03 pv_hash_locks+400      pmap_enter_ma+7d9
  0.05     142      0.51 pv_hash_locks          <all>
  0.05     140      0.49 pv_hash_locks          pmap_remove_pv+b5
  0.00       2      0.02 pv_hash_locks          pmap_enter_ma+7d9
  0.05     130      0.50 pv_hash_locks+3c0      <all>
  0.04     121      0.45 pv_hash_locks+3c0      pmap_remove_pv+b5
  0.01       9      0.06 pv_hash_locks+3c0      pmap_enter_ma+7d9
  0.05     143      0.49 pv_hash_locks+300      <all>
  0.05     139      0.49 pv_hash_locks+300      pmap_remove_pv+b5
  0.00       4      0.00 pv_hash_locks+300      pmap_enter_ma+7d9
  0.05     136      0.49 pv_hash_locks+2c0      <all>
  0.05     135      0.48 pv_hash_locks+2c0      pmap_remove_pv+b5
  0.00       1      0.01 pv_hash_locks+2c0      pmap_enter_ma+7d9
  0.05     132      0.48 pv_hash_locks+480      <all>
  0.05     128      0.48 pv_hash_locks+480      pmap_remove_pv+b5
  0.00       4      0.00 pv_hash_locks+480      pmap_enter_ma+7d9
  0.05     128      0.48 pv_hash_locks+500      <all>
  0.05     124      0.46 pv_hash_locks+500      pmap_remove_pv+b5
  0.00       4      0.02 pv_hash_locks+500      pmap_enter_ma+7d9
  0.05     140      0.47 pv_hash_locks+7c0      <all>
  0.05     137      0.47 pv_hash_locks+7c0      pmap_remove_pv+b5
  0.00       3      0.00 pv_hash_locks+7c0      pmap_enter_ma+7d9
  0.05     133      0.47 pv_hash_locks+180      <all>
  0.04     129      0.44 pv_hash_locks+180      pmap_remove_pv+b5
  0.00       4      0.02 pv_hash_locks+180      pmap_enter_ma+7d9
  0.05     137      0.46 pv_hash_locks+200      <all>
  0.04     133      0.44 pv_hash_locks+200      pmap_remove_pv+b5
  0.00       4      0.02 pv_hash_locks+200      pmap_enter_ma+7d9
  0.05     133      0.46 pv_hash_locks+140      <all>
  0.05     132      0.46 pv_hash_locks+140      pmap_remove_pv+b5
  0.00       1      0.00 pv_hash_locks+140      pmap_enter_ma+7d9
  0.05     126      0.46 pv_hash_locks+1c0      <all>
  0.04     121      0.44 pv_hash_locks+1c0      pmap_remove_pv+b5
  0.00       5      0.02 pv_hash_locks+1c0      pmap_enter_ma+7d9
  0.04     128      0.45 pv_hash_locks+100      <all>
  0.04     127      0.45 pv_hash_locks+100      pmap_remove_pv+b5
  0.00       1      0.00 pv_hash_locks+100      pmap_enter_ma+7d9
  0.04     128      0.45 pv_hash_locks+340      <all>
  0.04     127      0.44 pv_hash_locks+340      pmap_remove_pv+b5
  0.00       1      0.01 pv_hash_locks+340      pmap_enter_ma+7d9
  0.04     134      0.45 pv_hash_locks+80       <all>
  0.04     129      0.44 pv_hash_locks+80       pmap_remove_pv+b5
  0.00       5      0.01 pv_hash_locks+80       pmap_enter_ma+7d9
  0.04     127      0.44 pv_hash_locks+40       pmap_remove_pv+b5
  0.04     120      0.44 pv_hash_locks+4c0      <all>
  0.04     118      0.44 pv_hash_locks+4c0      pmap_remove_pv+b5
  0.00       2      0.00 pv_hash_locks+4c0      pmap_enter_ma+7d9
  0.04     124      0.44 pv_hash_locks+440      <all>
  0.04     123      0.43 pv_hash_locks+440      pmap_remove_pv+b5
  0.00       1      0.01 pv_hash_locks+440      pmap_enter_ma+7d9
  0.04     135      0.43 pv_hash_locks+680      <all>
  0.04     132      0.41 pv_hash_locks+680      pmap_remove_pv+b5
  0.00       3      0.02 pv_hash_locks+680      pmap_enter_ma+7d9
  0.04     128      0.42 pv_hash_locks+380      <all>
  0.04     123      0.39 pv_hash_locks+380      pmap_remove_pv+b5
  0.00       5      0.02 pv_hash_locks+380      pmap_enter_ma+7d9
  0.02     104      0.19 fffffe863e351940       <all>
  0.02     102      0.18 fffffe863e351940       turnstile_lookup+25
  0.00       2      0.01 fffffe863e351940       turnstile_block+3be
  0.01     156      0.14 fffffe8635a12b00       <all>
  0.01      81      0.08 fffffe8635a12b00       selnotify+3b
  0.00      56      0.05 fffffe8635a12b00       sel_do_scan+1cf
  0.00      19      0.02 fffffe8635a12b00       selclear+27
  0.01     136      0.13 fffffe8635a12d00       <all>
  0.01      61      0.07 fffffe8635a12d00       selnotify+3b
  0.00      57      0.05 fffffe8635a12d00       sel_do_scan+1cf
  0.00      17      0.02 fffffe8635a12d00       selclear+27
  0.00       1      0.00 fffffe8635a12d00       sched_pstats+c0
  0.01     140      0.13 fffffe8635a12900       <all>
  0.01      79      0.07 fffffe8635a12900       selnotify+3b
  0.00      50      0.04 fffffe8635a12900       sel_do_scan+1cf
  0.00      10      0.01 fffffe8635a12900       selclear+27
  0.00       1      0.00 fffffe8635a12900       sched_pstats+c0
  0.01     135      0.13 fffffe863e340140       <all>
  0.01      74      0.06 fffffe863e340140       selnotify+3b
  0.00      51      0.05 fffffe863e340140       sel_do_scan+1cf
  0.00      10      0.02 fffffe863e340140       selclear+27
  0.01     151      0.12 fffffe8635a12f00       <all>
  0.01      64      0.05 fffffe8635a12f00       sel_do_scan+1cf
  0.01      66      0.05 fffffe8635a12f00       selnotify+3b
  0.00      21      0.02 fffffe8635a12f00       selclear+27
  0.01     138      0.12 fffffe863e340740       <all>
  0.01      78      0.07 fffffe863e340740       selnotify+3b
  0.00      45      0.04 fffffe863e340740       sel_do_scan+1cf
  0.00      14      0.01 fffffe863e340740       selclear+27
  0.00       1      0.00 fffffe863e340740       sched_pstats+c0
  0.01      25      0.12 fffffe863e351c00       <all>
  0.01      24      0.11 fffffe863e351c00       turnstile_lookup+25
  0.00       1      0.00 fffffe863e351c00       turnstile_block+3be
  0.01     118      0.12 fffffe863e340cc0       <all>
  0.01      57      0.06 fffffe863e340cc0       selnotify+3b
  0.00      45      0.04 fffffe863e340cc0       sel_do_scan+1cf
  0.00      16      0.02 fffffe863e340cc0       selclear+27
  0.01      88      0.12 fffffe863e33f980       <all>
  0.01      76      0.11 fffffe863e33f980       cv_wait+9b
  0.00       9      0.01 fffffe863e33f980       cv_wakeup_all+45
  0.00       3      0.00 fffffe863e33f980       cv_wakeup_one+47
  0.01     138      0.12 fffffe8635a12c00       <all>
  0.01      61      0.05 fffffe8635a12c00       selnotify+3b
  0.00      54      0.04 fffffe8635a12c00       sel_do_scan+1cf
  0.00      22      0.02 fffffe8635a12c00       selclear+27
  0.00       1      0.00 fffffe8635a12c00       sched_pstats+c0
  0.01     123      0.11 fffffe8635a12e00       <all>
  0.01      70      0.07 fffffe8635a12e00       selnotify+3b
  0.00      41      0.04 fffffe8635a12e00       sel_do_scan+1cf
  0.00      12      0.01 fffffe8635a12e00       selclear+27
  0.01     108      0.11 fffffe863e341340       <all>
  0.01      66      0.06 fffffe863e341340       cv_wakeup_all+45
  0.00      42      0.05 fffffe863e341340       cv_timedwait_sig+a0
  0.01     165      0.11 fffffe863e340640       <all>
  0.00      73      0.05 fffffe863e340640       sel_do_scan+1cf
  0.00      67      0.04 fffffe863e340640       selnotify+3b
  0.00      25      0.02 fffffe863e340640       selclear+27
  0.01     119      0.11 fffffe863e340240       <all>
  0.01      58      0.06 fffffe863e340240       selnotify+3b
  0.00      51      0.04 fffffe863e340240       sel_do_scan+1cf
  0.00       9      0.01 fffffe863e340240       selclear+27
  0.00       1      0.00 fffffe863e340240       sleepq_timeout+16
  0.01     118      0.10 fffffe863e340340       <all>
  0.01      59      0.05 fffffe863e340340       selnotify+3b
  0.00      44      0.04 fffffe863e340340       sel_do_scan+1cf
  0.00      15      0.01 fffffe863e340340       selclear+27
  0.01     101      0.10 fffffe8635a12700       <all>
  0.01      61      0.06 fffffe8635a12700       selnotify+3b
  0.00      30      0.02 fffffe8635a12700       sel_do_scan+1cf
  0.00      10      0.01 fffffe8635a12700       selclear+27
  0.01     120      0.09 fffffe863e340a40       <all>
  0.00      57      0.05 fffffe863e340a40       selnotify+3b
  0.00      45      0.03 fffffe863e340a40       sel_do_scan+1cf
  0.00      18      0.02 fffffe863e340a40       selclear+27
  0.01      94      0.09 fffffe863e340540       <all>
  0.00      48      0.05 fffffe863e340540       selnotify+3b
  0.00      34      0.03 fffffe863e340540       sel_do_scan+1cf
  0.00      12      0.01 fffffe863e340540       selclear+27
  0.01     137      0.08 fffffe863e340840       <all>
  0.00      52      0.05 fffffe863e340840       selnotify+3b
  0.00      65      0.03 fffffe863e340840       sel_do_scan+1cf
  0.00      20      0.01 fffffe863e340840       selclear+27
  0.01     104      0.08 fffffe863e340440       <all>
  0.00      40      0.03 fffffe863e340440       selnotify+3b
  0.00      49      0.03 fffffe863e340440       sel_do_scan+1cf
  0.00      14      0.01 fffffe863e340440       selclear+27
  0.00       1      0.00 fffffe863e340440       sched_pstats+c0
  0.01      49      0.06 fffffe863e33f540       <all>
  0.00      23      0.03 fffffe863e33f540       cv_timedwait_sig+a0
  0.00      25      0.03 fffffe863e33f540       cv_wakeup_all+45
  0.00       1      0.00 fffffe863e33f540       sched_pstats+c0
  0.01      43      0.06 fffffe863e341200       <all>
  0.00      21      0.03 fffffe863e341200       cv_wakeup_all+45
  0.00      21      0.03 fffffe863e341200       cv_timedwait_sig+a0
  0.00       1      0.00 fffffe863e341200       sched_pstats+c0
  0.01      78      0.05 fffffe8635a12500       <all>
  0.00      35      0.02 fffffe8635a12500       sel_do_scan+1cf
  0.00      33      0.02 fffffe8635a12500       selnotify+3b
  0.00       9      0.00 fffffe8635a12500       selclear+27
  0.00       1      0.00 fffffe8635a12500       sched_pstats+c0
  0.00      18      0.04 fffffe863e3549c0       <all>
  0.00      17      0.04 fffffe863e3549c0       lwp_park+6a
  0.00       1      0.00 fffffe863e3549c0       sys__lwp_unpark_all+f7
  0.00      10      0.04 fffffe863e351640       turnstile_lookup+25
  0.00      36      0.03 fffffe8635a12600       <all>
  0.00      18      0.01 fffffe8635a12600       sel_do_scan+1cf
  0.00      12      0.01 fffffe8635a12600       selnotify+3b
  0.00       6      0.00 fffffe8635a12600       selclear+27
  0.00       9      0.03 fffffe863f3ac7c0       lwp_park+6a
  0.00       6      0.01 fffffe863e3413c0       cv_wakeup_all+45
  0.00       3      0.01 fffffe863e351700       turnstile_lookup+25
  0.00       8      0.01 fffffe863e33f240       cv_wakeup_all+45
  0.00       7      0.01 fffffe863e351a80       turnstile_lookup+25
  0.00       6      0.01 fffffe863e33f140       cv_wakeup_all+45
  0.00       6      0.01 fffffe863e340e00       cv_wakeup_all+45
  0.00       3      0.01 fffffe863e33f940       cv_wakeup_all+45
  0.00       3      0.01 fffffe863e341680       cv_wakeup_all+45
  0.00       4      0.00 fffffe8635a12dc0       <all>
  0.00       3      0.00 fffffe8635a12dc0       sched_pstats+c0
  0.00       1      0.00 fffffe8635a12dc0       cv_wait_sig+a6
  0.00       1      0.00 fffffe863e33f580       cv_wakeup_one+47
  0.00       6      0.00 fffffe863e341bc0       cv_wakeup_all+45
  0.00       2      0.00 fffffe863e340000       sched_pstats+c0
  0.00       3      0.00 fffffe863e33fb80       <all>
  0.00       2      0.00 fffffe863e33fb80       cv_wakeup_all+45
  0.00       1      0.00 fffffe863e33fb80       sched_pstats+c0
  0.00       2      0.00 fffffe863e340700       <all>
  0.00       1      0.00 fffffe863e340700       sched_pstats+c0
  0.00       1      0.00 fffffe863e340700       preempt+2a
  0.00       6      0.00 timer_lock             <all>
  0.00       2      0.00 timer_lock             timers_free+141
  0.00       2      0.00 timer_lock             dosetitimer+91
  0.00       2      0.00 timer_lock             timers_free+33
  0.00       2      0.00 fffffe863e33f1c0       cv_wakeup_all+45
  0.00       2      0.00 fffffe863e340800       sched_pstats+c0
  0.00       4      0.00 fffffe863e341b40       cv_wakeup_all+45
  0.00       1      0.00 fffffe8635a12cc0       sched_pstats+c0
  0.00       1      0.00 fffffe863e340d40       sched_pstats+c0
  0.00       2      0.00 fffffe863e340200       cv_wait_sig+a6
  0.00       2      0.00 fffffe863e33f8c0       cv_wakeup_all+45
  0.00       1      0.00 fffffe863e341dc0       cv_wait+9b
  0.00       1      0.00 fffffe863e351180       turnstile_lookup+25
  0.00       2      0.00 fffffe863e341440       cv_wakeup_all+45
  0.00       3      0.00 fffffe863e33f400       cv_wakeup_all+45
  0.00       1      0.00 fffffe8635a12ec0       sched_pstats+c0
  0.00       1      0.00 fffffe863e351c40       turnstile_lookup+25
  0.00       1      0.00 fffffe863e340900       sched_pstats+c0
  0.00       1      0.00 fffffe863e341580       cv_wakeup_all+45
  0.00       2      0.00 fffffe863e33f000       cv_wakeup_all+45
  0.00       1      0.00 fffffe863e33f800       cv_wakeup_all+45
  0.00       1      0.00 fffffe863e340180       callout_schedule+29

-- RW lock sleep (writer)

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 99.85      15  30931.65 fffffe82c0fc0ef8       genfs_lock+9f
  0.14     147     44.49 fffffe85aeebda98       vm_map_lock+67
  0.00      13      0.30 fffffe85aed94a70       genfs_lock+9f
  0.00      10      0.21 fffffe812ff7f2e0       genfs_lock+9f
  0.00       9      0.15 fffffe82ba798418       genfs_lock+9f
  0.00       2      0.13 fffffe82c0fc0a58       genfs_lock+9f
  0.00       7      0.13 fffffe82ba7989e0       genfs_lock+9f
  0.00       2      0.09 fffffe82c0fc06e0       genfs_lock+9f
  0.00       1      0.04 fffffe82c0fc0930       genfs_lock+9f
  0.00       4      0.04 fffffe82c1a3b990       genfs_lock+9f
  0.00       1      0.03 fffffe82c0fc0dd0       genfs_lock+9f
  0.00       1      0.03 fffffe82c0fbc110       genfs_lock+9f
  0.00       2      0.02 fffffe85aed94b98       genfs_lock+9f
  0.00       1      0.01 fffffe82c0fbc238       genfs_lock+9f

-- RW lock sleep (reader)

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 96.76     162      4.45 fffffe85aeebda98       uvm_fault_internal+151
  3.24       6      0.15 fffffe85aed94a70       genfs_lock+9f

-- RW lock spin

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 61.81      61      0.10 fffffe85aeebda98       <all>
 31.21      29      0.05 fffffe85aeebda98       vm_map_lock+67
 30.60      32      0.05 fffffe85aeebda98       uvm_fault_internal+151
 11.15      24      0.02 fffffe812ff7f2e0       genfs_lock+9f
  8.02      10      0.01 fffffe82c0fc0ef8       genfs_lock+9f
  6.24      17      0.01 fffffe82ba7989e0       genfs_lock+9f
  3.08      10      0.00 fffffe839cf7d5b0       genfs_lock+9f
  2.18      18      0.00 fffffe82ba798418       genfs_lock+9f
  1.87       6      0.00 fffffe85aed94a70       genfs_lock+9f
  1.61       3      0.00 fffffe82c0fc0930       genfs_lock+9f
  1.40       6      0.00 fffffe82c0fc06e0       genfs_lock+9f
  0.86       2      0.00 fffffe82c1a3b990       genfs_lock+9f
  0.71       2      0.00 fffffe85aed94b98       genfs_lock+9f
  0.66       2      0.00 fffffe82c0fc05b8       genfs_lock+9f
  0.18       1      0.00 fffffe82c0f82e08       genfs_lock+9f
  0.14       2      0.00 fffffe82c1a6fdd8       genfs_lock+9f
  0.10       2      0.00 fffffe82c1a3bab8       genfs_lock+9f

-- Kernel lock spin

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
100.00      10      0.07 kernel_lock            <all>
 92.23       1      0.06 kernel_lock            udp6_usrreq_wrapper+2d
  6.76       7      0.00 kernel_lock            VOP_POLL+40
  1.01       2      0.00 kernel_lock            VOP_LOCK+43

-- Kernel preemption defer

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
100.00      11   2454.28 in_critical_section    <all>
 50.15       2   1230.83 in_critical_section    ffffffff801143c5
 27.11       1    665.34 in_critical_section    uvm_page_locked_p+8
 22.74       1    558.09 in_critical_section    uvm_page_compare_key+11
  0.00       2      0.01 in_critical_section    mutex_oncpu+2e
  0.00       1      0.00 in_critical_section    mutex_oncpu+11
  0.00       1      0.00 in_critical_section    kpreempt_disabled
  0.00       1      0.00 in_critical_section    kpreempt_disable+27
  0.00       1      0.00 in_critical_section    mutex_oncpu+22
  0.00       1      0.00 in_critical_section    mutex_oncpu+47

--vkogqOf2sHV7VnPd
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="lockstat_32.txt"

transaction type: SELECT only
scaling factor: 800
query mode: simple
number of clients: 32
number of threads: 32
duration: 60 s
number of transactions actually processed: 259633
tps = 3264.730769 (including connections establishing)
tps = 3266.631230 (excluding connections establishing)
Elapsed time: 79.57 seconds.

-- Adaptive mutex spin

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 91.49 3877367 141862.19 fffffe81301694c0       <all>
 44.90 3877071  69617.33 fffffe81301694c0       uvm_fault_internal+62e
 32.24     188  49984.57 fffffe81301694c0       uvm_unmap_remove+245
 14.36     108  22260.29 fffffe81301694c0       uao_detach+1a
  7.35 1715930  11398.09 semlock                <all>
  6.26 1404651   9709.79 semlock                cv_wait_sig+10d
  1.09  311279   1688.30 semlock                sys_semop+d0
  0.71    6475   1096.63 fffffe81301696c0       <all>
  0.42      87    646.74 fffffe81301696c0       uvm_unmap_remove+245
  0.29    6388    449.90 fffffe81301696c0       uvm_fault_internal+62e
  0.32      61    490.48 fffffe8130169c00       <all>
  0.32      46    490.42 fffffe8130169c00       uvm_map_lock_entry+1d
  0.00      13      0.05 fffffe8130169c00       uvm_fault_internal+341
  0.00       2      0.01 fffffe8130169c00       amap_ref+1f
  0.06   31156     93.27 fffffe8635a124c0       <all>
  0.03   12406     43.61 fffffe8635a124c0       uvm_fault_internal+5c0
  0.02    9978     28.11 fffffe8635a124c0       ubc_release+ab
  0.01    8772     21.55 fffffe8635a124c0       ubc_alloc+74
  0.03   74234     45.48 uvm_pageqlock          <all>
  0.01   10096     19.32 uvm_pageqlock          uvm_anon_dispose+11c
  0.01   33350     13.33 uvm_pageqlock          uvm_fault_lower_enter+243
  0.00   16805      6.00 uvm_pageqlock          ubc_fault+303
  0.00    7235      3.38 uvm_pageqlock          uvm_fault_internal+e5f
  0.00    2738      1.12 uvm_pageqlock          genfs_getpages+8d3
  0.00    1696      0.98 uvm_pageqlock          uvm_fault_internal+4c7
  0.00    1391      0.79 uvm_pageqlock          uvm_fault_upper_enter.clone.4+1
  0.00     918      0.53 uvm_pageqlock          uvm_fault_internal+1160
  0.00       5      0.03 uvm_pageqlock          ufs_balloc_range+18f
  0.02    3841     24.50 fffffe812dc62100       <all>
  0.02    3820     24.05 fffffe812dc62100       uvm_fault_internal+341
  0.00       2      0.32 fffffe812dc62100       amap_unref+1f
  0.00       9      0.08 fffffe812dc62100       amap_copy+ab
  0.00      10      0.05 fffffe812dc62100       amap_ref+1f
  0.01     595     17.95 fffffe84f0438a80       <all>
  0.01     592     17.91 fffffe84f0438a80       uvm_fault_internal+62e
  0.00       1      0.03 fffffe84f0438a80       uvm_unmap_remove+245
  0.00       2      0.01 fffffe84f0438a80       vn_lock+c9
  0.00    1188      3.21 fffffe863f3acf40       <all>
  0.00     264      0.77 fffffe863f3acf40       sosend+4c
  0.00     226      0.73 fffffe863f3acf40       soreceive+db
  0.00     209      0.61 fffffe863f3acf40       sopoll+1c
  0.00     182      0.42 fffffe863f3acf40       sosend+2cd
  0.00      66      0.25 fffffe863f3acf40       soreceive+86c
  0.00     173      0.21 fffffe863f3acf40       ip6intr+13
  0.00      65      0.20 fffffe863f3acf40       cv_wait_sig+10d
  0.00       3      0.01 fffffe863f3acf40       ipintr+37
  0.00      85      2.87 fffffe812ebc8bc0       <all>
  0.00      82      2.86 fffffe812ebc8bc0       uvm_fault_internal+341
  0.00       3      0.01 fffffe812ebc8bc0       amap_ref+1f
  0.00     841      2.37 fffffe82c1a69e80       <all>
  0.00     280      0.71 fffffe82c1a69e80       vn_lock+c9
  0.00     194      0.61 fffffe82c1a69e80       ubc_alloc+5a8
  0.00     159      0.61 fffffe82c1a69e80       genfs_getpages+396
  0.00     160      0.30 fffffe82c1a69e80       ubc_fault+14b
  0.00      48      0.14 fffffe82c1a69e80       ubc_fault+1e2
  0.00     790      2.32 fffffe82c1a69ec0       <all>
  0.00     272      0.73 fffffe82c1a69ec0       vn_lock+c9
  0.00     199      0.60 fffffe82c1a69ec0       ubc_alloc+5a8
  0.00     110      0.45 fffffe82c1a69ec0       genfs_getpages+396
  0.00     162      0.36 fffffe82c1a69ec0       ubc_fault+14b
  0.00      47      0.17 fffffe82c1a69ec0       ubc_fault+1e2
  0.00     827      2.31 fffffe82c1a69f40       <all>
  0.00     282      0.79 fffffe82c1a69f40       vn_lock+c9
  0.00     214      0.59 fffffe82c1a69f40       ubc_alloc+5a8
  0.00     171      0.39 fffffe82c1a69f40       ubc_fault+14b
  0.00     108      0.39 fffffe82c1a69f40       genfs_getpages+396
  0.00      52      0.15 fffffe82c1a69f40       ubc_fault+1e2
  0.00     789      2.24 fffffe82c1a69f00       <all>
  0.00     286      0.76 fffffe82c1a69f00       vn_lock+c9
  0.00     189      0.51 fffffe82c1a69f00       ubc_alloc+5a8
  0.00     105      0.43 fffffe82c1a69f00       genfs_getpages+396
  0.00     151      0.33 fffffe82c1a69f00       ubc_fault+14b
  0.00      58      0.21 fffffe82c1a69f00       ubc_fault+1e2
  0.00     749      2.19 fffffe82c0fb0080       <all>
  0.00     252      0.71 fffffe82c0fb0080       vn_lock+c9
  0.00     194      0.54 fffffe82c0fb0080       ubc_alloc+5a8
  0.00     114      0.46 fffffe82c0fb0080       genfs_getpages+396
  0.00     145      0.35 fffffe82c0fb0080       ubc_fault+14b
  0.00      44      0.12 fffffe82c0fb0080       ubc_fault+1e2
  0.00     750      2.17 fffffe82c1a69f80       <all>
  0.00     280      0.76 fffffe82c1a69f80       vn_lock+c9
  0.00     175      0.48 fffffe82c1a69f80       ubc_alloc+5a8
  0.00     148      0.42 fffffe82c1a69f80       ubc_fault+14b
  0.00      98      0.37 fffffe82c1a69f80       genfs_getpages+396
  0.00      49      0.15 fffffe82c1a69f80       ubc_fault+1e2
  0.00     783      2.17 fffffe82c0fb0040       <all>
  0.00     290      0.78 fffffe82c0fb0040       vn_lock+c9
  0.00     169      0.50 fffffe82c0fb0040       ubc_alloc+5a8
  0.00     113      0.41 fffffe82c0fb0040       genfs_getpages+396
  0.00     154      0.31 fffffe82c0fb0040       ubc_fault+14b
  0.00      57      0.17 fffffe82c0fb0040       ubc_fault+1e2
  0.00     781      2.14 fffffe82c0fb00c0       <all>
  0.00     258      0.62 fffffe82c0fb00c0       vn_lock+c9
  0.00     137      0.51 fffffe82c0fb00c0       genfs_getpages+396
  0.00     182      0.51 fffffe82c0fb00c0       ubc_alloc+5a8
  0.00     158      0.34 fffffe82c0fb00c0       ubc_fault+14b
  0.00      45      0.16 fffffe82c0fb00c0       ubc_fault+1e2
  0.00       1      0.00 fffffe82c0fb00c0       vn_close+58
  0.00     741      2.06 fffffe82c0fb0000       <all>
  0.00     253      0.66 fffffe82c0fb0000       vn_lock+c9
  0.00     176      0.47 fffffe82c0fb0000       ubc_alloc+5a8
  0.00     107      0.40 fffffe82c0fb0000       genfs_getpages+396
  0.00     146      0.38 fffffe82c0fb0000       ubc_fault+14b
  0.00      59      0.16 fffffe82c0fb0000       ubc_fault+1e2
  0.00     675      1.87 fffffe82c0fb0100       <all>
  0.00     228      0.56 fffffe82c0fb0100       vn_lock+c9
  0.00     174      0.47 fffffe82c0fb0100       ubc_alloc+5a8
  0.00      94      0.38 fffffe82c0fb0100       genfs_getpages+396
  0.00     130      0.29 fffffe82c0fb0100       ubc_fault+14b
  0.00      49      0.17 fffffe82c0fb0100       ubc_fault+1e2
  0.00     298      1.21 xc_low_pri             <all>
  0.00     215      0.86 xc_low_pri             cv_wait+109
  0.00      58      0.26 xc_low_pri             xc_unicast+75
  0.00      14      0.07 xc_low_pri             xc_thread+88
  0.00      11      0.02 xc_low_pri             xc_wait+68
  0.00      57      0.51 fffffe8130169980       <all>
  0.00      51      0.41 fffffe8130169980       uvm_fault_internal+341
  0.00       2      0.05 fffffe8130169980       amap_ref+1f
  0.00       4      0.05 fffffe8130169980       amap_copy+ab
  0.00      49      0.30 fffffe812c98b400       <all>
  0.00      34      0.13 fffffe812c98b400       uvm_fault_internal+341
  0.00       8      0.09 fffffe812c98b400       amap_ref+1f
  0.00       5      0.07 fffffe812c98b400       amap_copy+ab
  0.00       2      0.01 fffffe812c98b400       amap_unref+1f
  0.00      15      0.12 fffffe8130169b80       <all>
  0.00       5      0.06 fffffe8130169b80       amap_ref+1f
  0.00       7      0.02 fffffe8130169b80       uvm_fault_internal+341
  0.00       1      0.02 fffffe8130169b80       amap_copy+ab
  0.00       1      0.01 fffffe8130169b80       amap_unref+1f
  0.00       1      0.01 fffffe8130169b80       uvm_map_lock_entry+1d
  0.00      13      0.10 fffffe8130169cc0       <all>
  0.00      10      0.07 fffffe8130169cc0       uvm_fault_internal+341
  0.00       1      0.01 fffffe8130169cc0       amap_copy+ab
  0.00       2      0.01 fffffe8130169cc0       amap_ref+1f
  0.00      94      0.09 fffffe82c1a69cc0       <all>
  0.00      92      0.09 fffffe82c1a69cc0       vn_lock+c9
  0.00       1      0.00 fffffe82c1a69cc0       genfs_getpages+396
  0.00       1      0.00 fffffe82c1a69cc0       ubc_fault+1e2
  0.00      14      0.07 fffffe8130169a40       <all>
  0.00       9      0.03 fffffe8130169a40       uvm_fault_internal+341
  0.00       2      0.03 fffffe8130169a40       amap_ref+1f
  0.00       3      0.02 fffffe8130169a40       amap_copy+ab
  0.00      81      0.06 fffffe82c1a69e40       <all>
  0.00      78      0.06 fffffe82c1a69e40       vn_lock+c9
  0.00       2      0.00 fffffe82c1a69e40       ubc_fault+1e2
  0.00       1      0.00 fffffe82c1a69e40       genfs_getpages+396
  0.00       1      0.06 fffffe863f3acf00       kill1.clone.0+8c
  0.00      15      0.05 fffffe812df27d80       <all>
  0.00       7      0.03 fffffe812df27d80       amap_copy+ab
  0.00       8      0.02 fffffe812df27d80       uvm_fault_internal+341
  0.00      34      0.05 fffffe85b6aaadc0       pmap_map_ptes+49
  0.00      11      0.03 fffffe812f148540       <all>
  0.00       9      0.03 fffffe812f148540       uvm_fault_internal+341
  0.00       2      0.01 fffffe812f148540       amap_copy+ab
  0.00       7      0.02 fffffe8130169d40       <all>
  0.00       6      0.02 fffffe8130169d40       uvm_fault_internal+341
  0.00       1      0.00 fffffe8130169d40       amap_ref+1f
  0.00       4      0.02 fffffe84f0438900       <all>
  0.00       2      0.01 fffffe84f0438900       uvm_fault_internal+62e
  0.00       1      0.01 fffffe84f0438900       uvm_unmap_remove+245
  0.00       1      0.00 fffffe84f0438900       vn_lock+c9
  0.00      36      0.02 fffffe85af4caec0       <all>
  0.00      12      0.01 fffffe85af4caec0       soreceive+db
  0.00       4      0.01 fffffe85af4caec0       cv_timedwait_sig+116
  0.00      10      0.00 fffffe85af4caec0       sopoll+1c
  0.00       5      0.00 fffffe85af4caec0       soreceive+86c
  0.00       1      0.00 fffffe85af4caec0       sosend+2cd
  0.00       4      0.00 fffffe85af4caec0       sosend+4c
  0.00      29      0.02 fffffe85b0184740       <all>
  0.00       8      0.01 fffffe85b0184740       soreceive+db
  0.00      11      0.00 fffffe85b0184740       sopoll+1c
  0.00       3      0.00 fffffe85b0184740       sosend+2cd
  0.00       2      0.00 fffffe85b0184740       sosend+4c
  0.00       2      0.00 fffffe85b0184740       soreceive+86c
  0.00       3      0.00 fffffe85b0184740       cv_timedwait_sig+116
  0.00       3      0.02 fffffe84f0438e80       uvm_fault_internal+62e
  0.00      39      0.02 fffffe85b28148c0       <all>
  0.00      10      0.01 fffffe85b28148c0       soreceive+86c
  0.00      10      0.00 fffffe85b28148c0       soreceive+db
  0.00      14      0.00 fffffe85b28148c0       sopoll+1c
  0.00       2      0.00 fffffe85b28148c0       cv_timedwait_sig+116
  0.00       3      0.00 fffffe85b28148c0       sosend+2cd
  0.00      38      0.02 fffffe82ba910740       <all>
  0.00      16      0.01 fffffe82ba910740       sopoll+1c
  0.00      10      0.00 fffffe82ba910740       soreceive+db
  0.00       5      0.00 fffffe82ba910740       soreceive+86c
  0.00       5      0.00 fffffe82ba910740       cv_timedwait_sig+116
  0.00       1      0.00 fffffe82ba910740       sosend+2cd
  0.00       1      0.00 fffffe82ba910740       sosend+4c
  0.00      22      0.02 fffffe8130169040       <all>
  0.00       8      0.01 fffffe8130169040       sopoll+1c
  0.00       9      0.00 fffffe8130169040       soreceive+db
  0.00       3      0.00 fffffe8130169040       cv_timedwait_sig+116
  0.00       1      0.00 fffffe8130169040       soreceive+86c
  0.00       1      0.00 fffffe8130169040       sosend+2cd
  0.00      35      0.02 fffffe82c1a695c0       <all>
  0.00      11      0.01 fffffe82c1a695c0       sopoll+1c
  0.00       4      0.00 fffffe82c1a695c0       soreceive+86c
  0.00      10      0.00 fffffe82c1a695c0       soreceive+db
  0.00       8      0.00 fffffe82c1a695c0       cv_timedwait_sig+116
  0.00       2      0.00 fffffe82c1a695c0       sosend+2cd
  0.00      29      0.02 fffffe85b1b42ac0       <all>
  0.00      11      0.01 fffffe85b1b42ac0       sopoll+1c
  0.00      11      0.00 fffffe85b1b42ac0       soreceive+db
  0.00       3      0.00 fffffe85b1b42ac0       soreceive+86c
  0.00       4      0.00 fffffe85b1b42ac0       cv_timedwait_sig+116
  0.00       4      0.02 fffffe82ba910e80       <all>
  0.00       3      0.01 fffffe82ba910e80       uvm_fault_internal+341
  0.00       1      0.01 fffffe82ba910e80       amap_copy+ab
  0.00      24      0.01 fffffe812c150a40       <all>
  0.00      12      0.01 fffffe812c150a40       sopoll+1c
  0.00       5      0.00 fffffe812c150a40       soreceive+86c
  0.00       1      0.00 fffffe812c150a40       uvm_fault_internal+341
  0.00       4      0.00 fffffe812c150a40       soreceive+db
  0.00       2      0.00 fffffe812c150a40       cv_timedwait_sig+116
  0.00      19      0.01 fffffe812da90740       <all>
  0.00       6      0.01 fffffe812da90740       sopoll+1c
  0.00       3      0.00 fffffe812da90740       soreceive+86c
  0.00       5      0.00 fffffe812da90740       soreceive+db
  0.00       4      0.00 fffffe812da90740       cv_timedwait_sig+116
  0.00       1      0.00 fffffe812da90740       sosend+4c
  0.00      34      0.01 fffffe85b1b42b00       <all>
  0.00      12      0.00 fffffe85b1b42b00       sopoll+1c
  0.00       8      0.00 fffffe85b1b42b00       soreceive+86c
  0.00       7      0.00 fffffe85b1b42b00       soreceive+db
  0.00       2      0.00 fffffe85b1b42b00       sosend+4c
  0.00       2      0.00 fffffe85b1b42b00       sosend+2cd
  0.00       3      0.00 fffffe85b1b42b00       cv_timedwait_sig+116
  0.00      30      0.01 fffffe85af0b01c0       <all>
  0.00      10      0.01 fffffe85af0b01c0       sopoll+1c
  0.00      11      0.00 fffffe85af0b01c0       soreceive+db
  0.00       2      0.00 fffffe85af0b01c0       soreceive+86c
  0.00       2      0.00 fffffe85af0b01c0       sosend+2cd
  0.00       4      0.00 fffffe85af0b01c0       cv_timedwait_sig+116
  0.00       1      0.00 fffffe85af0b01c0       sosend+4c
  0.00      25      0.01 fffffe86230fbcc0       <all>
  0.00       7      0.00 fffffe86230fbcc0       cv_timedwait_sig+116
  0.00       7      0.00 fffffe86230fbcc0       sopoll+1c
  0.00       6      0.00 fffffe86230fbcc0       soreceive+db
  0.00       3      0.00 fffffe86230fbcc0       soreceive+86c
  0.00       1      0.00 fffffe86230fbcc0       sosend+2cd
  0.00       1      0.00 fffffe86230fbcc0       sosend+4c
  0.00      22      0.01 fffffe862252e8c0       <all>
  0.00      11      0.01 fffffe862252e8c0       sopoll+1c
  0.00       8      0.00 fffffe862252e8c0       soreceive+db
  0.00       2      0.00 fffffe862252e8c0       soreceive+86c
  0.00       1      0.00 fffffe862252e8c0       cv_timedwait_sig+116
  0.00      20      0.01 fffffe85b0184d00       <all>
  0.00       9      0.01 fffffe85b0184d00       soreceive+db
  0.00       5      0.00 fffffe85b0184d00       sopoll+1c
  0.00       3      0.00 fffffe85b0184d00       cv_timedwait_sig+116
  0.00       1      0.00 fffffe85b0184d00       amap_copy+ab
  0.00       2      0.00 fffffe85b0184d00       soreceive+86c
  0.00      21      0.01 fffffe812dc62000       <all>
  0.00       3      0.00 fffffe812dc62000       cv_timedwait_sig+116
  0.00       7      0.00 fffffe812dc62000       soreceive+db
  0.00       9      0.00 fffffe812dc62000       sopoll+1c
  0.00       2      0.00 fffffe812dc62000       soreceive+86c
  0.00      25      0.01 fffffe85cb45f380       <all>
  0.00      10      0.00 fffffe85cb45f380       sopoll+1c
  0.00       4      0.00 fffffe85cb45f380       cv_timedwait_sig+116
  0.00       5      0.00 fffffe85cb45f380       soreceive+db
  0.00       5      0.00 fffffe85cb45f380       soreceive+86c
  0.00       1      0.00 fffffe85cb45f380       sosend+4c
  0.00      19      0.01 fffffe85b2814cc0       <all>
  0.00      10      0.01 fffffe85b2814cc0       sopoll+1c
  0.00       3      0.00 fffffe85b2814cc0       soreceive+db
  0.00       1      0.00 fffffe85b2814cc0       cv_timedwait_sig+116
  0.00       1      0.00 fffffe85b2814cc0       sosend+4c
  0.00       3      0.00 fffffe85b2814cc0       soreceive+86c
  0.00       1      0.00 fffffe85b2814cc0       sosend+2cd
  0.00      28      0.01 fffffe86230fbd00       <all>
  0.00      11      0.01 fffffe86230fbd00       soreceive+db
  0.00       9      0.00 fffffe86230fbd00       sopoll+1c
  0.00       3      0.00 fffffe86230fbd00       soreceive+86c
  0.00       3      0.00 fffffe86230fbd00       cv_timedwait_sig+116
  0.00       2      0.00 fffffe86230fbd00       sosend+4c
  0.00      27      0.01 fffffe85b9072e80       <all>
  0.00       6      0.00 fffffe85b9072e80       soreceive+86c
  0.00       3      0.00 fffffe85b9072e80       sosend+2cd
  0.00       6      0.00 fffffe85b9072e80       soreceive+db
  0.00       4      0.00 fffffe85b9072e80       sosend+4c
  0.00       4      0.00 fffffe85b9072e80       sopoll+1c
  0.00       4      0.00 fffffe85b9072e80       cv_timedwait_sig+116
  0.00      23      0.01 fffffe82ba9104c0       <all>
  0.00       9      0.00 fffffe82ba9104c0       sopoll+1c
  0.00       4      0.00 fffffe82ba9104c0       cv_timedwait_sig+116
  0.00       6      0.00 fffffe82ba9104c0       soreceive+db
  0.00       2      0.00 fffffe82ba9104c0       soreceive+86c
  0.00       1      0.00 fffffe82ba9104c0       sosend+4c
  0.00       1      0.00 fffffe82ba9104c0       sosend+2cd
  0.00      21      0.01 fffffe85af0b0240       <all>
  0.00      11      0.01 fffffe85af0b0240       sopoll+1c
  0.00       4      0.00 fffffe85af0b0240       soreceive+db
  0.00       3      0.00 fffffe85af0b0240       soreceive+86c
  0.00       1      0.00 fffffe85af0b0240       sosend+4c
  0.00       1      0.00 fffffe85af0b0240       sosend+2cd
  0.00       1      0.00 fffffe85af0b0240       cv_timedwait_sig+116
  0.00       4      0.01 fffffe837379fa80       uvm_fault_internal+62e
  0.00      21      0.01 fffffe85af0b0600       <all>
  0.00       5      0.00 fffffe85af0b0600       soreceive+86c
  0.00       7      0.00 fffffe85af0b0600       soreceive+db
  0.00       5      0.00 fffffe85af0b0600       sopoll+1c
  0.00       1      0.00 fffffe85af0b0600       sosend+2cd
  0.00       1      0.00 fffffe85af0b0600       sosend+4c
  0.00       2      0.00 fffffe85af0b0600       cv_timedwait_sig+116
  0.00      27      0.01 fffffe85b9072400       <all>
  0.00      10      0.00 fffffe85b9072400       sopoll+1c
  0.00       2      0.00 fffffe85b9072400       sosend+2cd
  0.00       7      0.00 fffffe85b9072400       soreceive+db
  0.00       6      0.00 fffffe85b9072400       cv_timedwait_sig+116
  0.00       1      0.00 fffffe85b9072400       sosend+4c
  0.00       1      0.00 fffffe85b9072400       soreceive+86c
  0.00       3      0.01 fffffe8130169b00       uvm_fault_internal+341
  0.00      22      0.01 fffffe86230fbd40       <all>
  0.00      10      0.00 fffffe86230fbd40       sopoll+1c
  0.00       7      0.00 fffffe86230fbd40       soreceive+db
  0.00       2      0.00 fffffe86230fbd40       soreceive+86c
  0.00       2      0.00 fffffe86230fbd40       cv_timedwait_sig+116
  0.00       1      0.00 fffffe86230fbd40       sosend+4c
  0.00      22      0.01 fffffe85b1b42300       <all>
  0.00       8      0.00 fffffe85b1b42300       sopoll+1c
  0.00       4      0.00 fffffe85b1b42300       soreceive+86c
  0.00       2      0.00 fffffe85b1b42300       sosend+4c
  0.00       4      0.00 fffffe85b1b42300       cv_timedwait_sig+116
  0.00       1      0.00 fffffe85b1b42300       sosend+2cd
  0.00       3      0.00 fffffe85b1b42300       soreceive+db
  0.00      18      0.01 fffffe82c1a69800       <all>
  0.00       7      0.00 fffffe82c1a69800       sopoll+1c
  0.00       7      0.00 fffffe82c1a69800       soreceive+db
  0.00       2      0.00 fffffe82c1a69800       soreceive+86c
  0.00       2      0.00 fffffe82c1a69800       cv_timedwait_sig+116
  0.00       3      0.01 fffffe8635a12440       <all>
  0.00       1      0.00 fffffe8635a12440       sopoll+1c
  0.00       1      0.00 fffffe8635a12440       sogetopt+1e
  0.00       1      0.00 fffffe8635a12440       do_sys_accept+9d
  0.00      25      0.01 fffffe85cb45f980       <all>
  0.00      10      0.00 fffffe85cb45f980       sopoll+1c
  0.00       5      0.00 fffffe85cb45f980       soreceive+86c
  0.00       5      0.00 fffffe85cb45f980       soreceive+db
  0.00       3      0.00 fffffe85cb45f980       sosend+4c
  0.00       2      0.00 fffffe85cb45f980       cv_timedwait_sig+116
  0.00      19      0.01 fffffe85b0184700       <all>
  0.00       9      0.00 fffffe85b0184700       sopoll+1c
  0.00       3      0.00 fffffe85b0184700       soreceive+db
  0.00       3      0.00 fffffe85b0184700       soreceive+86c
  0.00       2      0.00 fffffe85b0184700       sosend+4c
  0.00       1      0.00 fffffe85b0184700       cv_timedwait_sig+116
  0.00       1      0.00 fffffe85b0184700       sosend+2cd
  0.00      16      0.01 fffffe85b1b42740       <all>
  0.00       7      0.00 fffffe85b1b42740       soreceive+db
  0.00       2      0.00 fffffe85b1b42740       soreceive+86c
  0.00       4      0.00 fffffe85b1b42740       sopoll+1c
  0.00       3      0.00 fffffe85b1b42740       cv_timedwait_sig+116
  0.00      17      0.01 fffffe82c1a697c0       <all>
  0.00       6      0.00 fffffe82c1a697c0       soreceive+db
  0.00       3      0.00 fffffe82c1a697c0       soreceive+86c
  0.00       6      0.00 fffffe82c1a697c0       sopoll+1c
  0.00       2      0.00 fffffe82c1a697c0       cv_timedwait_sig+116
  0.00      18      0.01 fffffe812dc62200       <all>
  0.00       6      0.00 fffffe812dc62200       sopoll+1c
  0.00       6      0.00 fffffe812dc62200       soreceive+db
  0.00       2      0.00 fffffe812dc62200       sosend+2cd
  0.00       2      0.00 fffffe812dc62200       cv_timedwait_sig+116
  0.00       1      0.00 fffffe812dc62200       soreceive+86c
  0.00       1      0.00 fffffe812dc62200       sosend+4c
  0.00       3      0.00 fffffe84f0438b40       uvm_fault_internal+62e
  0.00      16      0.00 fffffe81301693c0       <all>
  0.00       8      0.00 fffffe81301693c0       vn_lock+115
  0.00       4      0.00 fffffe81301693c0       vn_lock+c9
  0.00       4      0.00 fffffe81301693c0       cache_lookup+112
  0.00       1      0.00 fffffe85b2814080       uvm_fault_internal+341
  0.00       5      0.00 uvm_anon_cache+b0      pool_get+54
  0.00       6      0.00 vnode_free_list_lock   vrelel+413
  0.00       1      0.00 fffffe85b2814300       uvm_fault_internal+341
  0.00       2      0.00 shm_lock               shmexit+20
  0.00       1      0.00 fffffe812da90840       uvm_fault_internal+341
  0.00       3      0.00 pmap_pv_cache+1a0      pool_cache_get_slow+217
  0.00       2      0.00 bufcache_lock          <all>
  0.00       1      0.00 bufcache_lock          brelse+19
  0.00       1      0.00 bufcache_lock          getblk+2c
  0.00       6      0.00 fffffe8130169280       pipe_poll+1a
  0.00       3      0.00 fffffe85af4cab00       <all>
  0.00       1      0.00 fffffe85af4cab00       lwp_exit+15a
  0.00       1      0.00 fffffe85af4cab00       lwp_exit+80
  0.00       1      0.00 fffffe85af4cab00       lwp_unpark+ad
  0.00       3      0.00 fffffe85b1b428c0       <all>
  0.00       2      0.00 fffffe85b1b428c0       sopoll+1c
  0.00       1      0.00 fffffe85b1b428c0       cv_timedwait_sig+116
  0.00       2      0.00 uvm_anon_cache+1a0     <all>
  0.00       1      0.00 uvm_anon_cache+1a0     pool_cache_get_slow+217
  0.00       1      0.00 uvm_anon_cache+1a0     pool_cache_put_slow+153
  0.00       1      0.00 fffffe85b1b42500       sopoll+1c
  0.00       1      0.00 fffffe8130169340       vn_lock+c9
  0.00       1      0.00 fffffe85b3554c80       fd_tryexpand+cf
  0.00       1      0.00 fffffe8635b0d370       pool_get+54
  0.00       1      0.00 fffffe863f3aa3a0       specificdata_fini+33
  0.00       1      0.00 fffffe863f3a15e8       lwp_ctl_alloc+8b

-- Adaptive mutex sleep

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 89.99     117   6621.33 fffffe81301694c0       <all>
 46.62      36   3430.52 fffffe81301694c0       uvm_unmap_remove+245
 28.78      69   2117.54 fffffe81301694c0       uvm_fault_internal+62e
 14.59      12   1073.27 fffffe81301694c0       uao_detach+1a
 10.01       5    736.55 fffffe8130169b80       uvm_map_lock_entry+1d
  0.00       5      0.06 semlock                cv_wait_sig+10d
  0.00       4      0.02 fffffe863f3acf40       <all>
  0.00       2      0.01 fffffe863f3acf40       sopoll+1c
  0.00       2      0.01 fffffe863f3acf40       route_intr+26
  0.00       1      0.01 uvm_pageqlock          uvm_fault_lower_enter+243
  0.00       1      0.01 fffffe8130169040       cv_timedwait_sig+116

-- Spin mutex spin

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 91.16  690337   1007.90 fffffe863e341700       <all>
 77.88  585801    861.12 fffffe863e341700       cv_wait_sig+99
 13.27  104516    146.68 fffffe863e341700       cv_wakeup_all+45
  0.01      20      0.10 fffffe863e341700       sched_pstats+c0
  4.22   37928     46.68 uvm_fpageqlock         <all>
  3.66   33236     40.52 uvm_fpageqlock         uvm_pagefree+224
  0.51    3941      5.69 uvm_fpageqlock         uvm_pagealloc_strat+c3
  0.04     724      0.42 uvm_fpageqlock         uvm_pageidlezero+201
  0.00      27      0.05 uvm_fpageqlock         uvm_pglistalloc+b8
  0.45     387      4.98 fffffe863e351800       <all>
  0.44     378      4.90 fffffe863e351800       turnstile_lookup+25
  0.01       7      0.06 fffffe863e351800       sched_pstats+c0
  0.00       2      0.02 fffffe863e351800       turnstile_block+3be
  0.21    7029      2.29 fffffe863e340d00       <all>
  0.16    5951      1.78 fffffe863e340d00       sched_idle+1ad
  0.02     430      0.23 fffffe863e340d00       sleepq_remove+107
  0.02     452      0.18 fffffe863e340d00       mi_switch+a9
  0.01     196      0.10 fffffe863e340d00       sched_idle+123
  0.19    5509      2.10 fffffe863e3401c0       <all>
  0.08    2350      0.91 fffffe863e3401c0       sched_idle+1ad
  0.05    2041      0.60 fffffe863e3401c0       sched_idle+11a
  0.02     493      0.27 fffffe863e3401c0       sleepq_remove+107
  0.02     376      0.20 fffffe863e3401c0       mi_switch+a9
  0.01     155      0.09 fffffe863e3401c0       sched_idle+1a3
  0.00      94      0.04 fffffe863e3401c0       sched_idle+123
  0.19    5198      2.05 fffffe8635a12f80       <all>
  0.08    2180      0.87 fffffe8635a12f80       sched_idle+1ad
  0.06    2031      0.70 fffffe8635a12f80       sched_idle+11a
  0.02     435      0.24 fffffe8635a12f80       sleepq_remove+107
  0.01     279      0.14 fffffe8635a12f80       mi_switch+a9
  0.01     176      0.08 fffffe8635a12f80       sched_idle+1a3
  0.00      97      0.03 fffffe8635a12f80       sched_idle+123
  0.18    5348      2.01 fffffe8635a12d80       <all>
  0.09    3107      1.01 fffffe8635a12d80       sched_idle+11a
  0.05    1229      0.50 fffffe8635a12d80       sched_idle+1ad
  0.02     446      0.25 fffffe8635a12d80       sleepq_remove+107
  0.01     357      0.16 fffffe8635a12d80       mi_switch+a9
  0.01     176      0.07 fffffe8635a12d80       sched_idle+1a3
  0.00      33      0.01 fffffe8635a12d80       sched_idle+123
  0.18    5427      1.96 fffffe863e3404c0       <all>
  0.10    3109      1.10 fffffe863e3404c0       sched_idle+1ad
  0.04    1326      0.42 fffffe863e3404c0       sched_idle+11a
  0.03     446      0.28 fffffe863e3404c0       sleepq_remove+107
  0.01     328      0.08 fffffe863e3404c0       mi_switch+a9
  0.00     130      0.05 fffffe863e3404c0       sched_idle+123
  0.00      87      0.03 fffffe863e3404c0       sched_idle+1a3
  0.00       1      0.00 fffffe863e3404c0       sched_pstats+c0
  0.18    4107      1.94 fffffe8635a12a80       <all>
  0.10    2396      1.06 fffffe8635a12a80       sched_idle+11a
  0.04     773      0.44 fffffe8635a12a80       sched_idle+1ad
  0.02     431      0.23 fffffe8635a12a80       sleepq_remove+107
  0.01     281      0.13 fffffe8635a12a80       mi_switch+a9
  0.01     190      0.08 fffffe8635a12a80       sched_idle+1a3
  0.00      36      0.01 fffffe8635a12a80       sched_idle+123
  0.17    5912      1.91 fffffe863e340ac0       <all>
  0.13    4698      1.42 fffffe863e340ac0       sched_idle+1ad
  0.02     395      0.21 fffffe863e340ac0       sleepq_remove+107
  0.01     368      0.14 fffffe863e340ac0       mi_switch+a9
  0.01     173      0.07 fffffe863e340ac0       sched_idle+123
  0.01     256      0.07 fffffe863e340ac0       sched_idle+11a
  0.00      22      0.01 fffffe863e340ac0       sched_idle+1a3
  0.17    3650      1.83 fffffe8635a12c80       <all>
  0.07    1474      0.79 fffffe8635a12c80       sched_idle+11a
  0.06    1281      0.64 fffffe8635a12c80       sched_idle+1ad
  0.02     405      0.22 fffffe8635a12c80       sleepq_remove+107
  0.01     264      0.10 fffffe8635a12c80       mi_switch+a9
  0.01     165      0.06 fffffe8635a12c80       sched_idle+1a3
  0.00      61      0.02 fffffe8635a12c80       sched_idle+123
  0.17    4953      1.83 fffffe863e3400c0       <all>
  0.09    2576      0.94 fffffe863e3400c0       sched_idle+1ad
  0.04    1412      0.48 fffffe863e3400c0       sched_idle+11a
  0.02     432      0.25 fffffe863e3400c0       sleepq_remove+107
  0.01     327      0.10 fffffe863e3400c0       mi_switch+a9
  0.00      96      0.03 fffffe863e3400c0       sched_idle+123
  0.00     110      0.03 fffffe863e3400c0       sched_idle+1a3
  0.15    4632      1.71 fffffe863e3405c0       <all>
  0.09    2892      1.02 fffffe863e3405c0       sched_idle+1ad
  0.02     931      0.26 fffffe863e3405c0       sched_idle+11a
  0.02     350      0.21 fffffe863e3405c0       sleepq_remove+107
  0.01     277      0.14 fffffe863e3405c0       mi_switch+a9
  0.00     115      0.05 fffffe863e3405c0       sched_idle+123
  0.00      67      0.03 fffffe863e3405c0       sched_idle+1a3
  0.15    4353      1.70 fffffe8635a12980       <all>
  0.08    2696      0.88 fffffe8635a12980       sched_idle+11a
  0.03     501      0.33 fffffe8635a12980       sleepq_remove+107
  0.02     612      0.21 fffffe8635a12980       sched_idle+1ad
  0.01     285      0.14 fffffe8635a12980       mi_switch+a9
  0.01     223      0.13 fffffe8635a12980       sched_idle+1a3
  0.00      36      0.01 fffffe8635a12980       sched_idle+123
  0.15    3628      1.64 fffffe8635a12680       <all>
  0.11    2577      1.16 fffffe8635a12680       sched_idle+11a
  0.02     435      0.26 fffffe8635a12680       sleepq_remove+107
  0.01     231      0.11 fffffe8635a12680       mi_switch+a9
  0.01     195      0.07 fffffe8635a12680       sched_idle+1a3
  0.00     184      0.04 fffffe8635a12680       sched_idle+1ad
  0.00       6      0.00 fffffe8635a12680       sched_idle+123
  0.15    3503      1.63 fffffe8635a12780       <all>
  0.09    2169      0.96 fffffe8635a12780       sched_idle+11a
  0.02     454      0.21 fffffe8635a12780       sched_idle+1ad
  0.02     393      0.21 fffffe8635a12780       sleepq_remove+107
  0.01     239      0.14 fffffe8635a12780       mi_switch+a9
  0.01     218      0.10 fffffe8635a12780       sched_idle+1a3
  0.00      30      0.01 fffffe8635a12780       sched_idle+123
  0.14    5834      1.59 fffffe863e3408c0       <all>
  0.09    3970      1.04 fffffe863e3408c0       sched_idle+1ad
  0.02     420      0.23 fffffe863e3408c0       sleepq_remove+107
  0.02     776      0.22 fffffe863e3408c0       sched_idle+11a
  0.01     430      0.06 fffffe863e3408c0       mi_switch+a9
  0.00     178      0.03 fffffe863e3408c0       sched_idle+123
  0.00      60      0.01 fffffe863e3408c0       sched_idle+1a3
  0.14    4623      1.55 fffffe863e3407c0       <all>
  0.09    3001      0.96 fffffe863e3407c0       sched_idle+1ad
  0.02     806      0.21 fffffe863e3407c0       sched_idle+11a
  0.02     359      0.20 fffffe863e3407c0       sleepq_remove+107
  0.01     287      0.11 fffffe863e3407c0       mi_switch+a9
  0.00     117      0.05 fffffe863e3407c0       sched_idle+123
  0.00      53      0.02 fffffe863e3407c0       sched_idle+1a3
  0.14    3594      1.54 fffffe8635a12880       <all>
  0.08    2414      0.93 fffffe8635a12880       sched_idle+11a
  0.02     460      0.27 fffffe8635a12880       sleepq_remove+107
  0.01     284      0.14 fffffe8635a12880       mi_switch+a9
  0.01     205      0.11 fffffe8635a12880       sched_idle+1a3
  0.01     220      0.08 fffffe8635a12880       sched_idle+1ad
  0.00      11      0.01 fffffe8635a12880       sched_idle+123
  0.13    3598      1.47 fffffe8635a12580       <all>
  0.10    2633      1.06 fffffe8635a12580       sched_idle+11a
  0.02     461      0.27 fffffe8635a12580       sleepq_remove+107
  0.01     287      0.09 fffffe8635a12580       mi_switch+a9
  0.01     217      0.06 fffffe8635a12580       sched_idle+1a3
  0.13    3292      1.44 fffffe8635a12e80       <all>
  0.05    1457      0.58 fffffe8635a12e80       sched_idle+11a
  0.04     966      0.41 fffffe8635a12e80       sched_idle+1ad
  0.02     378      0.21 fffffe8635a12e80       sleepq_remove+107
  0.01     260      0.12 fffffe8635a12e80       mi_switch+a9
  0.01     166      0.08 fffffe8635a12e80       sched_idle+1a3
  0.00      65      0.04 fffffe8635a12e80       sched_idle+123
  0.13    4396      1.42 fffffe863e3409c0       <all>
  0.08    3285      0.91 fffffe863e3409c0       sched_idle+1ad
  0.02     299      0.19 fffffe863e3409c0       sleepq_remove+107
  0.01     416      0.14 fffffe863e3409c0       sched_idle+11a
  0.01     240      0.11 fffffe863e3409c0       mi_switch+a9
  0.01     124      0.06 fffffe863e3409c0       sched_idle+123
  0.00      32      0.01 fffffe863e3409c0       sched_idle+1a3
  0.10    2708      1.12 fffffe8635a12b80       <all>
  0.05    1382      0.53 fffffe8635a12b80       sched_idle+11a
  0.03     712      0.29 fffffe8635a12b80       sched_idle+1ad
  0.01     280      0.16 fffffe8635a12b80       sleepq_remove+107
  0.01     187      0.08 fffffe8635a12b80       mi_switch+a9
  0.00     117      0.04 fffffe8635a12b80       sched_idle+1a3
  0.00      29      0.01 fffffe8635a12b80       sched_idle+123
  0.00       1      0.00 fffffe8635a12b80       sched_pstats+216
  0.04    1263      0.49 fffffe863e3406c0       <all>
  0.02     691      0.26 fffffe863e3406c0       sched_idle+1ad
  0.01     319      0.13 fffffe863e3406c0       sched_idle+11a
  0.01     110      0.07 fffffe863e3406c0       sleepq_remove+107
  0.00      97      0.02 fffffe863e3406c0       mi_switch+a9
  0.00      23      0.01 fffffe863e3406c0       sched_idle+123
  0.00      23      0.00 fffffe863e3406c0       sched_idle+1a3
  0.04     932      0.40 fffffe863e3402c0       <all>
  0.02     550      0.25 fffffe863e3402c0       sched_idle+1ad
  0.01     207      0.07 fffffe863e3402c0       sched_idle+11a
  0.00      63      0.03 fffffe863e3402c0       sleepq_remove+107
  0.00      65      0.03 fffffe863e3402c0       mi_switch+a9
  0.00      23      0.01 fffffe863e3402c0       sched_idle+1a3
  0.00      24      0.00 fffffe863e3402c0       sched_idle+123
  0.03     113      0.32 pv_hash_locks+600      <all>
  0.03     109      0.31 pv_hash_locks+600      pmap_remove_pv+b5
  0.00       4      0.01 pv_hash_locks+600      pmap_enter_ma+7d9
  0.03     110      0.30 pv_hash_locks+540      <all>
  0.03     107      0.30 pv_hash_locks+540      pmap_remove_pv+b5
  0.00       3      0.01 pv_hash_locks+540      pmap_enter_ma+7d9
  0.03     111      0.30 pv_hash_locks+5c0      <all>
  0.03     108      0.29 pv_hash_locks+5c0      pmap_remove_pv+b5
  0.00       3      0.01 pv_hash_locks+5c0      pmap_enter_ma+7d9
  0.03     114      0.29 pv_hash_locks+300      <all>
  0.03     111      0.29 pv_hash_locks+300      pmap_remove_pv+b5
  0.00       3      0.00 pv_hash_locks+300      pmap_enter_ma+7d9
  0.03     102      0.29 pv_hash_locks+580      <all>
  0.03      99      0.29 pv_hash_locks+580      pmap_remove_pv+b5
  0.00       3      0.00 pv_hash_locks+580      pmap_enter_ma+7d9
  0.03     111      0.29 pv_hash_locks+340      <all>
  0.03     110      0.29 pv_hash_locks+340      pmap_remove_pv+b5
  0.00       1      0.00 pv_hash_locks+340      pmap_enter_ma+7d9
  0.03     780      0.28 fffffe863e3403c0       <all>
  0.01     415      0.14 fffffe863e3403c0       sched_idle+1ad
  0.01     203      0.06 fffffe863e3403c0       sched_idle+11a
  0.00      71      0.04 fffffe863e3403c0       sleepq_remove+107
  0.00      54      0.02 fffffe863e3403c0       mi_switch+a9
  0.00      18      0.01 fffffe863e3403c0       sched_idle+123
  0.00      19      0.01 fffffe863e3403c0       sched_idle+1a3
  0.03     111      0.28 pv_hash_locks+400      <all>
  0.03     105      0.28 pv_hash_locks+400      pmap_remove_pv+b5
  0.00       6      0.00 pv_hash_locks+400      pmap_enter_ma+7d9
  0.03     114      0.28 pv_hash_locks+380      <all>
  0.02     112      0.27 pv_hash_locks+380      pmap_remove_pv+b5
  0.00       2      0.01 pv_hash_locks+380      pmap_enter_ma+7d9
  0.02     115      0.27 pv_hash_locks+640      <all>
  0.02     107      0.26 pv_hash_locks+640      pmap_remove_pv+b5
  0.00       8      0.01 pv_hash_locks+640      pmap_enter_ma+7d9
  0.02      98      0.27 pv_hash_locks+6c0      <all>
  0.02      95      0.26 pv_hash_locks+6c0      pmap_remove_pv+b5
  0.00       3      0.01 pv_hash_locks+6c0      pmap_enter_ma+7d9
  0.02     107      0.27 pv_hash_locks+500      <all>
  0.02     103      0.26 pv_hash_locks+500      pmap_remove_pv+b5
  0.00       4      0.00 pv_hash_locks+500      pmap_enter_ma+7d9
  0.02      90      0.27 pv_hash_locks+780      <all>
  0.02      87      0.26 pv_hash_locks+780      pmap_remove_pv+b5
  0.00       3      0.00 pv_hash_locks+780      pmap_enter_ma+7d9
  0.02     100      0.27 pv_hash_locks+2c0      <all>
  0.02      99      0.27 pv_hash_locks+2c0      pmap_remove_pv+b5
  0.00       1      0.00 pv_hash_locks+2c0      pmap_enter_ma+7d9
  0.02     108      0.26 pv_hash_locks+3c0      <all>
  0.02     103      0.25 pv_hash_locks+3c0      pmap_remove_pv+b5
  0.00       5      0.01 pv_hash_locks+3c0      pmap_enter_ma+7d9
  0.02      93      0.26 pv_hash_locks          <all>
  0.02      91      0.26 pv_hash_locks          pmap_remove_pv+b5
  0.00       2      0.00 pv_hash_locks          pmap_enter_ma+7d9
  0.02     102      0.25 pv_hash_locks+180      <all>
  0.02      97      0.25 pv_hash_locks+180      pmap_remove_pv+b5
  0.00       5      0.01 pv_hash_locks+180      pmap_enter_ma+7d9
  0.02      93      0.25 pv_hash_locks+680      <all>
  0.02      88      0.25 pv_hash_locks+680      pmap_remove_pv+b5
  0.00       5      0.01 pv_hash_locks+680      pmap_enter_ma+7d9
  0.02     115      0.25 pv_hash_locks+4c0      <all>
  0.02     112      0.25 pv_hash_locks+4c0      pmap_remove_pv+b5
  0.00       3      0.00 pv_hash_locks+4c0      pmap_enter_ma+7d9
  0.02      86      0.24 pv_hash_locks+7c0      <all>
  0.02      83      0.23 pv_hash_locks+7c0      pmap_remove_pv+b5
  0.00       3      0.01 pv_hash_locks+7c0      pmap_enter_ma+7d9
  0.02      87      0.24 pv_hash_locks+100      <all>
  0.02      83      0.24 pv_hash_locks+100      pmap_remove_pv+b5
  0.00       4      0.00 pv_hash_locks+100      pmap_enter_ma+7d9
  0.02     106      0.24 pv_hash_locks+440      <all>
  0.02     102      0.24 pv_hash_locks+440      pmap_remove_pv+b5
  0.00       4      0.00 pv_hash_locks+440      pmap_enter_ma+7d9
  0.02      93      0.24 pv_hash_locks+700      <all>
  0.02      86      0.23 pv_hash_locks+700      pmap_remove_pv+b5
  0.00       7      0.02 pv_hash_locks+700      pmap_enter_ma+7d9
  0.02     103      0.24 pv_hash_locks+480      <all>
  0.02     102      0.24 pv_hash_locks+480      pmap_remove_pv+b5
  0.00       1      0.00 pv_hash_locks+480      pmap_enter_ma+7d9
  0.02      86      0.24 pv_hash_locks+200      <all>
  0.02      85      0.24 pv_hash_locks+200      pmap_remove_pv+b5
  0.00       1      0.00 pv_hash_locks+200      pmap_enter_ma+7d9
  0.02      88      0.23 pv_hash_locks+240      <all>
  0.02      87      0.23 pv_hash_locks+240      pmap_remove_pv+b5
  0.00       1      0.01 pv_hash_locks+240      pmap_enter_ma+7d9
  0.02      91      0.23 pv_hash_locks+280      <all>
  0.02      89      0.23 pv_hash_locks+280      pmap_remove_pv+b5
  0.00       2      0.00 pv_hash_locks+280      pmap_enter_ma+7d9
  0.02      88      0.23 pv_hash_locks+740      <all>
  0.02      86      0.23 pv_hash_locks+740      pmap_remove_pv+b5
  0.00       2      0.00 pv_hash_locks+740      pmap_enter_ma+7d9
  0.02      84      0.22 pv_hash_locks+140      <all>
  0.02      79      0.21 pv_hash_locks+140      pmap_remove_pv+b5
  0.00       5      0.01 pv_hash_locks+140      pmap_enter_ma+7d9
  0.02      82      0.22 pv_hash_locks+40       <all>
  0.02      80      0.21 pv_hash_locks+40       pmap_remove_pv+b5
  0.00       2      0.01 pv_hash_locks+40       pmap_enter_ma+7d9
  0.02      81      0.21 pv_hash_locks+80       <all>
  0.02      77      0.21 pv_hash_locks+80       pmap_remove_pv+b5
  0.00       4      0.00 pv_hash_locks+80       pmap_enter_ma+7d9
  0.02      79      0.21 pv_hash_locks+1c0      <all>
  0.02      77      0.21 pv_hash_locks+1c0      pmap_remove_pv+b5
  0.00       2      0.00 pv_hash_locks+1c0      pmap_enter_ma+7d9
  0.02      96      0.20 pv_hash_locks+c0       <all>
  0.02      94      0.20 pv_hash_locks+c0       pmap_remove_pv+b5
  0.00       2      0.00 pv_hash_locks+c0       pmap_enter_ma+7d9
  0.02      36      0.18 fffffe863e351c00       <all>
  0.02      34      0.17 fffffe863e351c00       turnstile_lookup+25
  0.00       2      0.01 fffffe863e351c00       turnstile_block+3be
  0.02     112      0.17 fffffe863e3516c0       turnstile_lookup+25
  0.01     133      0.12 fffffe8635a12c00       <all>
  0.01      69      0.07 fffffe8635a12c00       selnotify+3b
  0.00      47      0.03 fffffe8635a12c00       sel_do_scan+1cf
  0.00      17      0.01 fffffe8635a12c00       selclear+27
  0.01      70      0.10 fffffe863e33f980       <all>
  0.01      55      0.08 fffffe863e33f980       cv_wait+9b
  0.00      13      0.01 fffffe863e33f980       cv_wakeup_all+45
  0.00       2      0.00 fffffe863e33f980       cv_wakeup_one+47
  0.01      98      0.09 fffffe8635a12800       <all>
  0.00      48      0.05 fffffe8635a12800       selnotify+3b
  0.00      38      0.03 fffffe8635a12800       sel_do_scan+1cf
  0.00      12      0.01 fffffe8635a12800       selclear+27
  0.01      94      0.08 fffffe8635a12d00       <all>
  0.00      44      0.04 fffffe8635a12d00       selnotify+3b
  0.00      35      0.02 fffffe8635a12d00       sel_do_scan+1cf
  0.00      14      0.01 fffffe8635a12d00       selclear+27
  0.00       1      0.00 fffffe8635a12d00       sched_pstats+c0
  0.01      79      0.08 fffffe863e340140       <all>
  0.00      35      0.04 fffffe863e340140       selnotify+3b
  0.00      30      0.03 fffffe863e340140       sel_do_scan+1cf
  0.00      14      0.01 fffffe863e340140       selclear+27
  0.01      97      0.08 fffffe863e340040       <all>
  0.00      45      0.05 fffffe863e340040       selnotify+3b
  0.00      41      0.03 fffffe863e340040       sel_do_scan+1cf
  0.00      11      0.01 fffffe863e340040       selclear+27
  0.01      87      0.08 fffffe8635a12600       <all>
  0.00      41      0.04 fffffe8635a12600       selnotify+3b
  0.00      29      0.02 fffffe8635a12600       sel_do_scan+1cf
  0.00      17      0.01 fffffe8635a12600       selclear+27
  0.01      73      0.07 fffffe8635a12900       <all>
  0.00      31      0.04 fffffe8635a12900       selnotify+3b
  0.00      27      0.02 fffffe8635a12900       sel_do_scan+1cf
  0.00      15      0.01 fffffe8635a12900       selclear+27
  0.01      73      0.07 fffffe863e340a40       <all>
  0.00      40      0.04 fffffe863e340a40       selnotify+3b
  0.00      27      0.02 fffffe863e340a40       sel_do_scan+1cf
  0.00       6      0.01 fffffe863e340a40       selclear+27
  0.01      75      0.07 fffffe8635a12f00       <all>
  0.00      35      0.03 fffffe8635a12f00       sel_do_scan+1cf
  0.00      28      0.02 fffffe8635a12f00       selnotify+3b
  0.00      12      0.01 fffffe8635a12f00       selclear+27
  0.01      80      0.07 fffffe8635a12e00       <all>
  0.00      34      0.03 fffffe8635a12e00       selnotify+3b
  0.00      36      0.03 fffffe8635a12e00       sel_do_scan+1cf
  0.00      10      0.01 fffffe8635a12e00       selclear+27
  0.01      76      0.06 fffffe8635a12700       <all>
  0.00      42      0.03 fffffe8635a12700       selnotify+3b
  0.00      24      0.02 fffffe8635a12700       sel_do_scan+1cf
  0.00      10      0.01 fffffe8635a12700       selclear+27
  0.01      67      0.06 fffffe8635a12a00       <all>
  0.00      39      0.03 fffffe8635a12a00       selnotify+3b
  0.00      21      0.02 fffffe8635a12a00       sel_do_scan+1cf
  0.00       7      0.01 fffffe8635a12a00       selclear+27
  0.01      54      0.06 fffffe863e340540       <all>
  0.00      23      0.03 fffffe863e340540       selnotify+3b
  0.00      23      0.02 fffffe863e340540       sel_do_scan+1cf
  0.00       8      0.01 fffffe863e340540       selclear+27
  0.01      62      0.06 fffffe8635a12500       <all>
  0.00      38      0.04 fffffe8635a12500       selnotify+3b
  0.00       9      0.01 fffffe8635a12500       sel_do_scan+1cf
  0.00      14      0.01 fffffe8635a12500       selclear+27
  0.00       1      0.00 fffffe8635a12500       sleepq_timeout+16
  0.00      66      0.05 fffffe863e340440       <all>
  0.00      36      0.03 fffffe863e340440       selnotify+3b
  0.00      20      0.01 fffffe863e340440       sel_do_scan+1cf
  0.00      10      0.01 fffffe863e340440       selclear+27
  0.00      64      0.05 fffffe863e340840       <all>
  0.00      33      0.03 fffffe863e340840       selnotify+3b
  0.00      23      0.01 fffffe863e340840       sel_do_scan+1cf
  0.00       8      0.01 fffffe863e340840       selclear+27
  0.00      53      0.05 fffffe863e340740       <all>
  0.00      23      0.02 fffffe863e340740       selnotify+3b
  0.00      25      0.02 fffffe863e340740       sel_do_scan+1cf
  0.00       5      0.01 fffffe863e340740       selclear+27
  0.00      38      0.04 fffffe863e340940       <all>
  0.00      20      0.03 fffffe863e340940       selnotify+3b
  0.00      12      0.01 fffffe863e340940       sel_do_scan+1cf
  0.00       6      0.00 fffffe863e340940       selclear+27
  0.00      31      0.03 fffffe863e340cc0       <all>
  0.00      14      0.02 fffffe863e340cc0       selnotify+3b
  0.00      12      0.01 fffffe863e340cc0       sel_do_scan+1cf
  0.00       4      0.00 fffffe863e340cc0       selclear+27
  0.00       1      0.00 fffffe863e340cc0       sched_pstats+c0
  0.00      12      0.03 fffffe863f3ac540       lwp_park+6a
  0.00      41      0.03 fffffe8635a12b00       <all>
  0.00      19      0.02 fffffe8635a12b00       selnotify+3b
  0.00      13      0.01 fffffe8635a12b00       sel_do_scan+1cf
  0.00       9      0.00 fffffe8635a12b00       selclear+27
  0.00      34      0.03 fffffe863e340240       <all>
  0.00      16      0.01 fffffe863e340240       selnotify+3b
  0.00      13      0.01 fffffe863e340240       sel_do_scan+1cf
  0.00       5      0.00 fffffe863e340240       selclear+27
  0.00      17      0.03 fffffe863e341580       <all>
  0.00       9      0.02 fffffe863e341580       cv_wakeup_all+45
  0.00       8      0.01 fffffe863e341580       cv_timedwait_sig+a0
  0.00      30      0.02 fffffe863e3412c0       <all>
  0.00      19      0.01 fffffe863e3412c0       cv_wakeup_all+45
  0.00      11      0.01 fffffe863e3412c0       cv_timedwait_sig+a0
  0.00      23      0.02 fffffe863e341a80       <all>
  0.00       9      0.01 fffffe863e341a80       cv_timedwait_sig+a0
  0.00      14      0.01 fffffe863e341a80       cv_wakeup_all+45
  0.00      18      0.02 fffffe863e340340       <all>
  0.00      10      0.01 fffffe863e340340       selnotify+3b
  0.00       7      0.01 fffffe863e340340       sel_do_scan+1cf
  0.00       1      0.00 fffffe863e340340       selclear+27
  0.00      26      0.02 fffffe863e33f240       <all>
  0.00      11      0.01 fffffe863e33f240       cv_wakeup_all+45
  0.00      15      0.01 fffffe863e33f240       cv_wait_sig+99
  0.00       9      0.01 fffffe863e354400       lwp_park+6a
  0.00       4      0.01 fffffe863e340ec0       cv_wakeup_all+45
  0.00       5      0.01 fffffe863e354440       lwp_park+6a
  0.00      20      0.01 fffffe863e340640       <all>
  0.00      13      0.01 fffffe863e340640       sel_do_scan+1cf
  0.00       5      0.00 fffffe863e340640       selnotify+3b
  0.00       2      0.00 fffffe863e340640       selclear+27
  0.00       7      0.01 fffffe863e341100       cv_wakeup_all+45
  0.00       5      0.01 fffffe863e33f400       cv_wakeup_all+45
  0.00       5      0.01 fffffe863e33fb00       cv_wakeup_all+45
  0.00       3      0.01 fffffe863e341d00       cv_wakeup_all+45
  0.00       5      0.00 fffffe863e341740       cv_wakeup_all+45
  0.00       3      0.00 fffffe863e341bc0       cv_wakeup_all+45
  0.00       5      0.00 fffffe863e33ff80       cv_wakeup_all+45
  0.00       5      0.00 fffffe863e340d80       cv_wakeup_all+45
  0.00       3      0.00 fffffe863e341a00       cv_wakeup_all+45
  0.00       1      0.00 fffffe863e351480       turnstile_lookup+25
  0.00       6      0.00 fffffe863e341440       cv_wakeup_all+45
  0.00       2      0.00 fffffe863e351c40       turnstile_lookup+25
  0.00       4      0.00 fffffe863e33fcc0       cv_wakeup_all+45
  0.00       3      0.00 fffffe863e33fb80       cv_wakeup_all+45
  0.00       5      0.00 fffffe863e33f540       cv_wakeup_all+45
  0.00       1      0.00 fffffe863e351a80       turnstile_lookup+25
  0.00       4      0.00 timer_lock             <all>
  0.00       3      0.00 timer_lock             timers_free+141
  0.00       1      0.00 timer_lock             timers_free+33
  0.00       4      0.00 fffffe863e33f140       cv_wakeup_all+45
  0.00       4      0.00 fffffe863e341d80       cv_wakeup_all+45
  0.00       3      0.00 fffffe863e3413c0       cv_wakeup_all+45
  0.00       3      0.00 fffffe863e33fa40       cv_wakeup_all+45
  0.00       2      0.00 fffffe863e341b40       cv_wakeup_all+45
  0.00       3      0.00 fffffe863e3419c0       cv_wait+9b
  0.00       1      0.00 fffffe863e341c40       cv_wakeup_all+45
  0.00       2      0.00 fffffe863e33fe40       cv_wakeup_all+45
  0.00       1      0.00 fffffe863e3415c0       cv_wait+9b
  0.00       1      0.00 fffffe863e340900       cv_wait_sig+a6
  0.00       1      0.00 fffffe863e340800       cv_wait_sig+a6
  0.00       1      0.00 fffffe8635a126c0       preempt+2a
  0.00       1      0.00 fffffe863e340a80       callout_schedule+29
  0.00       1      0.00 fffffe863e340880       callout_schedule+29

-- RW lock sleep (writer)

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 91.07     152      3.85 fffffe812c1618e8       vm_map_lock+67
  3.24       1      0.14 fffffe85aed94b98       genfs_lock+9f
  2.30       6      0.10 fffffe85aed94a70       genfs_lock+9f
  1.04       2      0.04 fffffe82c0fc0ca8       genfs_lock+9f
  0.77       2      0.03 fffffe82ba798418       genfs_lock+9f
  0.42       1      0.02 fffffe82c0fc0b80       genfs_lock+9f
  0.36       1      0.02 fffffe82c0fc0a58       genfs_lock+9f
  0.32       2      0.01 fffffe82c0fbc238       genfs_lock+9f
  0.30       1      0.01 fffffe82c0fc06e0       genfs_lock+9f
  0.17       1      0.01 fffffe82ba7989e0       genfs_lock+9f

-- RW lock sleep (reader)

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 99.15     186      3.94 fffffe812c1618e8       uvm_fault_internal+151
  0.85       1      0.03 fffffe85aed94b98       genfs_lock+9f

-- RW lock spin

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
 77.22      60      0.09 fffffe812c1618e8       <all>
 59.81      37      0.07 fffffe812c1618e8       uvm_fault_internal+151
 17.42      23      0.02 fffffe812c1618e8       vm_map_lock+67
  6.20      11      0.01 fffffe839cf7d5b0       genfs_lock+9f
  3.50       8      0.00 fffffe82ba798418       genfs_lock+9f
  3.37      11      0.00 fffffe812ff7f2e0       genfs_lock+9f
  1.48       9      0.00 fffffe82ba7989e0       genfs_lock+9f
  1.22       2      0.00 fffffe82c0fc06e0       genfs_lock+9f
  1.13       3      0.00 fffffe82c0fc0930       genfs_lock+9f
  1.05       2      0.00 fffffe82c0fc0808       genfs_lock+9f
  0.89       2      0.00 fffffe82c0fc0dd0       genfs_lock+9f
  0.79       1      0.00 fffffe82c0fc0240       genfs_lock+9f
  0.77       1      0.00 fffffe82c0fc05b8       genfs_lock+9f
  0.70       2      0.00 fffffe82c0fc0ca8       genfs_lock+9f
  0.50       1      0.00 fffffe82c0fbc110       genfs_lock+9f
  0.45       2      0.00 fffffe82c0fc0b80       genfs_lock+9f
  0.22       1      0.00 fffffe82c0fc0a58       genfs_lock+9f
  0.15       1      0.00 fffffe82c0fc0ef8       genfs_lock+9f
  0.11       1      0.00 fffffe85aed94b98       genfs_lock+9f
  0.10       1      0.00 fffffe82c0f82e08       genfs_lock+9f
  0.10       1      0.00 fffffe85aed94a70       genfs_lock+9f
  0.04       1      0.00 fffffe82c0fbc238       genfs_lock+9f

-- Kernel lock spin

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
100.00     202      0.45 kernel_lock            <all>
 18.74      59      0.08 kernel_lock            VOP_POLL+40
 17.40       1      0.08 kernel_lock            fileassoc_file_delete+20
 15.79       1      0.07 kernel_lock            udp6_usrreq_wrapper+2d
 10.12       1      0.05 kernel_lock            kevent1+613
  7.54      34      0.03 kernel_lock            ipintr+41
  6.88      28      0.03 kernel_lock            VOP_LOCK+43
  6.74      23      0.03 kernel_lock            intr_biglock_wrapper+16
  4.23      16      0.02 kernel_lock            VOP_IOCTL+4c
  3.39      10      0.02 kernel_lock            tcp_input+15d1
  3.05       4      0.01 kernel_lock            tcp_usrreq_wrapper+2d
  2.09      15      0.01 kernel_lock            VOP_UNLOCK+40
  1.93       3      0.01 kernel_lock            ip6intr+1d
  1.84       2      0.01 kernel_lock            sleepq_block+1f1
  0.18       2      0.00 kernel_lock            VOP_WRITE+48
  0.05       2      0.00 kernel_lock            VOP_READ+48
  0.03       1      0.00 kernel_lock            cdev_read+4d

-- Kernel preemption defer

Total%  Count   Time/ms          Lock                       Caller
------ ------- --------- ---------------------- ------------------------------
100.00      22   1877.00 in_critical_section    <all>
 63.31      12   1188.40 in_critical_section    ffffffff801143c5
 13.73       1    257.80 in_critical_section    uvm_page_locked_p+8
  9.00       2    169.02 in_critical_section    ffffffff80892ff6
  7.61       1    142.88 in_critical_section    pmap_remove_pte+1
  5.92       1    111.20 in_critical_section    mutex_owned+34
  0.41       2      7.70 in_critical_section    kpreempt_disabled
  0.00       1      0.00 in_critical_section    kpreempt_disable+27
  0.00       1      0.00 in_critical_section    mutex_oncpu+5
  0.00       1      0.00 in_critical_section    kpreempt_disabled+23

--vkogqOf2sHV7VnPd--





More information about the Kernel mailing list