Skip to content

Instantly share code, notes, and snippets.

@mcavage
Created December 8, 2011 01:52
Show Gist options
  • Select an option

  • Save mcavage/1445737 to your computer and use it in GitHub Desktop.

Select an option

Save mcavage/1445737 to your computer and use it in GitHub Desktop.

Revisions

  1. mcavage created this gist Dec 8, 2011.
    1,051 changes: 1,051 additions & 0 deletions gistfile1.txt
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,1051 @@
    [root@headnode (bh1-kvm1:0) ~]# dtrace -n 'profile-997 /execname == "beam.smp"/ { @[stack()] = count(); }'
    dtrace: description 'profile-997 ' matched 1 probe
    ^C


    genunix`writev+0x248
    unix`sys_syscall+0x17a
    1

    genunix`syscall_mstate+0x91
    unix`_sys_sysenter_post_swapgs+0x165
    1

    ip`tcp_sendmsg+0x4f
    sockfs`so_sendmsg+0x21b
    sockfs`socket_sendmsg+0x61
    sockfs`socket_vop_write+0x63
    genunix`fop_write+0xd3
    genunix`writev+0x41a
    unix`sys_syscall+0x17a
    1

    sockfs`copyout_name
    sockfs`recv+0x7a
    unix`sys_syscall+0x17a
    1

    unix`mul32
    genunix`scalehrtime+0x19
    genunix`mstate_thread_onproc_time+0x78
    unix`caps_charge_adjust+0x2d
    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`fop_getpage+0x49
    genunix`segvn_fault+0xae1
    genunix`as_fault+0x5ee
    unix`pagefault+0x99
    unix`trap+0xe63
    unix`0xfffffffffb8001d6
    1

    FSS`fss_active+0x1
    genunix`sleepq_wakeone_chan+0x89
    genunix`cv_signal+0x8e
    genunix`lwp_unpark+0x62
    genunix`syslwp_park+0x42
    unix`sys_syscall+0x17a
    1

    unix`mul32+0x6
    genunix`scalehrtime+0x19
    genunix`cpu_update_pct+0xc5
    unix`setbackdq+0x2b3
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`cv_broadcast+0x12
    poll`dpioctl+0x683
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    unix`cpu_wakeup_mwait+0x2b
    unix`setfrontdq+0x3f2
    FSS`fss_wakeup+0x9e
    genunix`sleepq_wakeone_chan+0x89
    genunix`cv_signal+0x8e
    genunix`lwp_unpark+0x62
    genunix`syslwp_park+0x42
    unix`sys_syscall+0x17a
    1

    unix`mul32+0xd
    genunix`scalehrtime+0x19
    genunix`cpu_update_pct+0xc5
    unix`setbackdq+0x2b3
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    unix`mutex_enter+0x10
    zfs`dbuf_hold_impl+0xea
    zfs`dbuf_hold+0x2e
    zfs`dmu_buf_hold_array_by_dnode+0x1a7
    zfs`dmu_read+0xd9
    zfs`zfs_fillpage+0x11b
    zfs`zfs_getpage+0x15e
    genunix`fop_getpage+0x9a
    genunix`segvn_fault+0xae1
    genunix`as_fault+0x5ee
    unix`pagefault+0x99
    unix`trap+0xe63
    unix`0xfffffffffb8001d6
    1

    unix`mutex_enter+0x10
    genunix`ioctl+0x205
    unix`sys_syscall+0x17a
    1

    unix`mutex_enter+0x10
    genunix`syslwp_park+0x42
    unix`sys_syscall+0x17a
    1

    unix`mutex_enter+0x10
    genunix`idtot_and_lock+0x24
    genunix`lwp_unpark+0x34
    genunix`syslwp_park+0x42
    unix`sys_syscall+0x17a
    1

    unix`mutex_enter+0x10
    sockfs`so_sendmsg+0x21b
    sockfs`socket_sendmsg+0x61
    sockfs`socket_vop_write+0x63
    genunix`fop_write+0xd3
    genunix`writev+0x41a
    unix`sys_syscall+0x17a
    1

    genunix`lwp_hash_lookup_and_lock+0xaa
    genunix`idtot_and_lock+0x24
    genunix`lwp_unpark+0x34
    genunix`syslwp_park+0x42
    unix`_sys_sysenter_post_swapgs+0x149
    1

    genunix`copymsg+0x6d
    mac`mac_tx_send+0x30f
    mac`mac_tx_single_ring_mode+0x91
    mac`mac_tx+0x35d
    dld`str_mdata_fastpath_put+0xa4
    ip`ip_xmit+0x7eb
    ip`ire_send_wire_v4+0x345
    ip`conn_ip_output+0x250
    ip`tcp_output+0x86d
    ip`squeue_enter+0x440
    ip`tcp_sendmsg+0x104
    sockfs`so_sendmsg+0x21b
    sockfs`socket_sendmsg+0x61
    sockfs`socket_vop_write+0x63
    genunix`fop_write+0xd3
    genunix`writev+0x41a
    unix`sys_syscall+0x17a
    1

    unix`tsc_gethrtime
    unix`pc_gethrestime+0x49
    genunix`gethrestime+0x19
    unix`gethrestime_sec+0x11
    fifofs`fifo_read+0x2c0
    genunix`fop_read+0xd3
    genunix`read+0x2b8
    unix`sys_syscall+0x17a
    1

    unix`bitset_in_set+0x1
    unix`setbackdq+0x468
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`_sys_sysenter_post_swapgs+0x149
    1

    genunix`syscall_mstate+0x1b3
    unix`sys_syscall+0x10e
    1

    unix`sys_syscall+0x121
    1

    unix`tsc_scalehrtime+0x14
    genunix`scalehrtime+0x19
    genunix`cpu_update_pct+0xc5
    unix`setbackdq+0x2b3
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`bt_getlowbit+0x75
    poll`dp_pcache_poll+0x104
    poll`dpioctl+0x2dd
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    unix`tsc_scalehrtime+0x1c
    genunix`scalehrtime+0x19
    genunix`mstate_thread_onproc_time+0x78
    unix`caps_charge_adjust+0x2d
    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`hwblkclr+0x24
    unix`pfnzero+0xd7
    unix`pagezero+0x32
    genunix`anon_zero+0xc3
    genunix`segvn_faultpage+0x2a4
    genunix`segvn_fault+0xc13
    genunix`as_fault+0x5ee
    unix`pagefault+0x99
    unix`trap+0xe63
    unix`0xfffffffffb8001d6
    1

    genunix`lbolt_event_driven+0xfd
    ip`ire_send_wire_v4+0x345
    ip`conn_ip_output+0x250
    ip`tcp_output+0x86d
    ip`squeue_enter+0x440
    ip`tcp_sendmsg+0x104
    sockfs`so_sendmsg+0x21b
    sockfs`socket_sendmsg+0x61
    sockfs`socket_vop_write+0x63
    genunix`fop_write+0xd3
    genunix`writev+0x41a
    unix`sys_syscall+0x17a
    1

    poll`dpioctl+0x336
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    genunix`fop_ioctl+0x85
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    tmpfs`tmp_close+0x40
    genunix`fop_close+0x71
    genunix`closef+0x5f
    genunix`closeandsetf+0x4f5
    genunix`close+0x18
    unix`sys_syscall+0x17a
    1

    unix`tsc_read
    genunix`gethrtime_unscaled+0xd
    genunix`syscall_mstate+0x5c
    unix`sys_syscall+0x10e
    1

    unix`tsc_read+0x3
    genunix`gethrtime_unscaled+0xd
    genunix`mstate_thread_onproc_time+0x53
    unix`caps_charge_adjust+0x2d
    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`tsc_read+0x3
    genunix`gethrtime_unscaled+0xd
    genunix`syscall_mstate+0x5c
    unix`_sys_sysenter_post_swapgs+0x106
    1

    unix`do_splx+0x8d
    genunix`disp_lock_exit_nopreempt+0x43
    genunix`yield+0x52
    unix`_sys_sysenter_post_swapgs+0x149
    1

    unix`do_splx+0x8d
    unix`cbe_restore_level+0x17
    genunix`cyclic_reprogram_cyclic+0xfd
    genunix`cyclic_reprogram+0xa4
    genunix`callout_heap_insert+0x6f
    genunix`timeout_generic+0x333
    genunix`cv_timedwait_sig_hires+0xf9
    genunix`cv_reltimedwait_sig+0x53
    genunix`cv_relwaituntil_sig+0x25
    poll`dpioctl+0x31b
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    genunix`untimeout_generic
    genunix`cv_timedwait_sig_hires+0x20b
    genunix`cv_reltimedwait_sig+0x53
    genunix`cv_relwaituntil_sig+0x25
    poll`dpioctl+0x31b
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    poll`dpwrite+0x79
    genunix`cdev_write+0x3d
    specfs`spec_write+0x48c
    genunix`fop_write+0xd3
    genunix`write+0x2e2
    unix`sys_syscall+0x17a
    1

    unix`setbackdq+0x83
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`bt_getlowbit+0x9f
    poll`dp_pcache_poll+0x104
    poll`dpioctl+0x2dd
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    unix`page_lookup_create+0xd0
    unix`page_lookup+0x26
    genunix`swap_getapage+0xaf
    genunix`swap_getpage+0x85
    genunix`fop_getpage+0x9a
    genunix`anon_zero+0xa3
    genunix`segvn_faultpage+0x2a4
    genunix`segvn_fault+0xc13
    genunix`as_fault+0x5ee
    unix`pagefault+0x99
    unix`trap+0xe63
    unix`0xfffffffffb8001d6
    1

    genunix`cv_wait_sig_swap_core
    genunix`cv_waituntil_sig+0x13c
    genunix`lwp_park+0x157
    genunix`syslwp_park+0x31
    unix`sys_syscall+0x17a
    1

    genunix`lwp_unpark
    unix`sys_syscall+0x17a
    1

    genunix`ioctl+0x1b
    unix`sys_syscall+0x17a
    1

    genunix`fop_rwlock
    genunix`read32+0x22
    unix`_sys_sysenter_post_swapgs+0x149
    1

    unix`hwblkclr+0x5b
    unix`pfnzero+0xd7
    unix`pagezero+0x32
    genunix`anon_zero+0xc3
    genunix`segvn_faultpage+0x2a4
    genunix`segvn_fault+0xc13
    genunix`as_fault+0x5ee
    unix`pagefault+0x99
    unix`trap+0xe63
    unix`0xfffffffffb8001d6
    1

    ip`ip_output_cksum_v4+0xde
    ip`ire_send_wire_v4+0x2b3
    ip`conn_ip_output+0x250
    ip`tcp_output+0x86d
    ip`squeue_enter+0x440
    ip`tcp_sendmsg+0x104
    sockfs`so_sendmsg+0x21b
    sockfs`socket_sendmsg+0x61
    sockfs`socket_vop_write+0x63
    genunix`fop_write+0xd3
    genunix`writev+0x41a
    unix`sys_syscall+0x17a
    1

    unix`splr+0x8
    genunix`thread_lock+0x1d
    genunix`yield+0x23
    unix`sys_syscall+0x17a
    1

    unix`setfrontdq+0x209
    FSS`fss_wakeup+0x9e
    genunix`sleepq_wakeone_chan+0x89
    genunix`cv_signal+0x8e
    genunix`pollnotify+0x40
    genunix`pollwakeup+0x117
    genunix`strpollwakeup+0x1d
    fifofs`fifo_wakereader+0x4e
    fifofs`fifo_write+0x31e
    genunix`fop_write+0xd3
    genunix`write+0x2e2
    unix`sys_syscall+0x17a
    1

    unix`tsc_scalehrtime+0x5b
    genunix`mstate_thread_onproc_time+0x78
    unix`caps_charge_adjust+0x2d
    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`new_mstate+0x1ac
    genunix`lwp_park+0x286
    genunix`syslwp_park+0x31
    unix`sys_syscall+0x17a
    1

    unix`cpucaps_charge+0xbe
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`post_syscall+0x77f
    unix`0xfffffffffb800c91
    1

    unix`splr+0x13
    genunix`thread_lock+0x1d
    genunix`yield+0x23
    unix`sys_syscall+0x17a
    1

    unix`setbackdq+0xb5
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`cpu_update_pct+0xc5
    unix`setbackdq+0x2b3
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`setbackdq+0x2ba
    FSS`fss_wakeup+0x94
    genunix`sleepq_wakeone_chan+0x89
    genunix`cv_signal+0x8e
    genunix`lwp_unpark+0x62
    genunix`syslwp_park+0x42
    unix`sys_syscall+0x17a
    1

    unix`tsc_gethrtimeunscaled+0xe
    genunix`gethrtime_unscaled+0xd
    genunix`syscall_mstate+0x5c
    unix`sys_syscall+0x1a1
    1

    genunix`syscall_mstate+0x10
    unix`sys_syscall+0x10e
    1

    genunix`syscall_mstate+0x10
    unix`_sys_sysenter_post_swapgs+0x106
    1

    genunix`segvn_faultpage+0x87c
    genunix`segvn_fault+0xc13
    genunix`as_fault+0x5ee
    unix`pagefault+0x99
    unix`trap+0xe63
    unix`0xfffffffffb8001d6
    1

    unix`swtch+0x6
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    unix`pg_null_op
    unix`swtch+0x17a
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    genunix`write+0x279
    unix`sys_syscall+0x17a
    1

    unix`fpxsave_ctxt+0x23
    unix`resume+0x5b
    unix`swtch+0x145
    genunix`cv_timedwait_sig_hires+0x1e9
    genunix`cv_reltimedwait_sig+0x53
    genunix`cv_relwaituntil_sig+0x25
    poll`dpioctl+0x31b
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`_sys_sysenter_post_swapgs+0x149
    1

    unix`disp_ratify+0x34
    unix`disp+0x198
    unix`swtch+0xa4
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    genunix`syscall_mstate+0x221
    unix`sys_syscall+0x10e
    1

    genunix`syscall_mstate+0x221
    unix`_sys_sysenter_post_swapgs+0x106
    1

    genunix`syscall_mstate+0x22
    unix`sys_syscall+0x1a1
    1

    unix`bzero+0x35b
    zfs`zio_null+0x4c
    zfs`zio_root+0x35
    zfs`dmu_buf_hold_array_by_dnode+0x167
    zfs`dmu_read+0xd9
    zfs`zfs_fillpage+0x11b
    zfs`zfs_getpage+0x15e
    genunix`fop_getpage+0x9a
    genunix`segvn_fault+0xae1
    genunix`as_fault+0x5ee
    unix`pagefault+0x99
    unix`trap+0xe63
    unix`0xfffffffffb8001d6
    1

    genunix`disp_lock_enter_high+0x15
    unix`caps_charge_adjust+0x6a
    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`membar_enter+0x3
    unix`setbackdq+0x449
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`membar_enter+0x3
    unix`disp+0x198
    unix`swtch+0xa4
    genunix`yield+0x57
    unix`_sys_sysenter_post_swapgs+0x149
    1

    unix`membar_enter+0x3
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`_sys_sysenter_post_swapgs+0x149
    1

    unix`pg_ev_thread_remain+0xbe
    unix`swtch+0x17a
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    genunix`mstate_thread_onproc_time+0x78
    unix`caps_charge_adjust+0x2d
    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`lock_try
    genunix`yield+0x23
    unix`sys_syscall+0x17a
    1

    genunix`disp_lock_exit_high+0x8
    genunix`thread_transition+0x2a
    genunix`yield+0x33
    unix`sys_syscall+0x17a
    1

    unix`disp+0xb3
    unix`swtch+0xa4
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    unix`atomic_cas_32+0x6
    unix`setbackdq+0x2b3
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`cpucaps_charge
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`lock_try+0x8
    genunix`yield+0x23
    unix`sys_syscall+0x17a
    1

    genunix`disp_lock_exit_nopreempt
    unix`sys_syscall+0x17a
    1

    genunix`list_is_empty+0x1
    sockfs`socket_poll+0x2a
    sockfs`socket_vop_poll+0x29
    genunix`fop_poll+0x3a
    poll`dpwrite+0x2e1
    genunix`cdev_write+0x3d
    specfs`spec_write+0x48c
    genunix`fop_write+0xd3
    genunix`write+0x2e2
    unix`sys_syscall+0x17a
    1

    genunix`fop_ioctl+0x1
    unix`sys_syscall+0x17a
    1

    unix`disp_getwork+0x12a
    unix`disp+0x1c2
    unix`swtch+0xa4
    genunix`cv_timedwait_sig_hires+0x1e9
    genunix`cv_reltimedwait_sig+0x53
    genunix`cv_relwaituntil_sig+0x25
    poll`dpioctl+0x31b
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    unix`cpu_resched+0xeb
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`new_mstate+0x1f7
    genunix`lwp_park+0x286
    genunix`syslwp_park+0x31
    unix`sys_syscall+0x17a
    1

    genunix`fop_ioctl+0x8
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    genunix`mstate_thread_onproc_time+0x8b
    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`list_is_empty+0xc
    sockfs`so_poll+0xaf
    sockfs`socket_poll+0x2a
    sockfs`socket_vop_poll+0x29
    genunix`fop_poll+0x3a
    poll`dp_pcache_poll+0x1ff
    poll`dpioctl+0x2dd
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`_sys_sysenter_post_swapgs+0x149
    1

    unix`cpucaps_charge+0xc
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    fifofs`fifo_read+0x2e6
    genunix`fop_read+0xd3
    genunix`read+0x2b8
    unix`sys_syscall+0x17a
    1

    genunix`cv_wait_sig_swap_core+0x6f
    genunix`cv_wait_sig_swap+0x18
    genunix`cv_waituntil_sig+0x13c
    genunix`lwp_park+0x157
    genunix`syslwp_park+0x31
    unix`sys_syscall+0x17a
    1

    genunix`swapfs_getvp+0x1
    genunix`anon_zero+0x71
    genunix`segvn_faultpage+0x2a4
    genunix`segvn_fault+0xc13
    genunix`as_fault+0x5ee
    unix`pagefault+0x99
    unix`trap+0xe63
    unix`0xfffffffffb8001d6
    1

    genunix`disp_lock_exit_high+0x24
    genunix`thread_transition+0x2a
    genunix`yield+0x33
    unix`sys_syscall+0x17a
    1

    unix`setbackdq+0x30c
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    unix`cpu_resched+0xc
    unix`setbackdq+0x449
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`new_mstate+0x216
    genunix`cv_block+0x8f
    genunix`cv_wait_sig_swap_core+0xb5
    genunix`cv_wait_sig_swap+0x18
    genunix`cv_waituntil_sig+0x13c
    genunix`lwp_park+0x157
    genunix`syslwp_park+0x31
    unix`sys_syscall+0x17a
    1

    genunix`callout_list_get+0x2f
    genunix`timeout_generic+0x245
    ip`tcp_timeout+0xa5
    ip`tcp_output+0x332
    ip`squeue_enter+0x440
    ip`tcp_sendmsg+0x104
    sockfs`so_sendmsg+0x21b
    sockfs`socket_sendmsg+0x61
    sockfs`socket_vop_write+0x63
    genunix`fop_write+0xd3
    genunix`writev+0x41a
    unix`sys_syscall+0x17a
    1

    genunix`pcache_lookup_fd+0x18
    poll`dp_pcache_poll+0x184
    poll`dpioctl+0x2dd
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    unix`pg_ev_thread_swtch+0x3c
    unix`swtch+0xdb
    genunix`cv_timedwait_sig_hires+0x1e9
    genunix`cv_reltimedwait_sig+0x53
    genunix`cv_relwaituntil_sig+0x25
    poll`dpioctl+0x31b
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    unix`atomic_add_64+0x4
    unix`sys_syscall+0x10e
    1

    unix`atomic_add_64+0x4
    unix`swtch+0xfb
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    genunix`writev+0x12c
    unix`sys_syscall+0x17a
    1

    genunix`disp_lock_enter
    unix`swtch+0xa4
    genunix`yield+0x57
    unix`_sys_sysenter_post_swapgs+0x149
    1

    unix`disp_ratify+0x8c
    unix`disp+0x198
    unix`swtch+0xa4
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    unix`cpu_resched+0x1f
    unix`setbackdq+0x449
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`crgetmapped+0x19
    genunix`fop_write+0xb7
    genunix`writev+0x41a
    unix`sys_syscall+0x17a
    1

    unix`lwp_getdatamodel+0x1a
    genunix`post_syscall+0x67b
    genunix`syscall_exit+0x59
    unix`0xfffffffffb800ec9
    1

    unix`disp_ratify+0x96
    unix`disp+0x198
    unix`swtch+0xa4
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    unix`mutex_delay_default+0x7
    unix`mutex_vector_enter+0x2ad
    genunix`pollwakeup+0x1cb
    genunix`strpollwakeup+0x1d
    fifofs`fifo_wakereader+0x4e
    fifofs`fifo_write+0x31e
    genunix`fop_write+0xd3
    genunix`write+0x2e2
    unix`sys_syscall+0x17a
    1

    unix`mutex_delay_default+0x7
    unix`lock_set_spin+0xad
    genunix`disp_lock_enter_high+0x15
    unix`caps_charge_adjust+0x6a
    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    1

    genunix`writev+0x142
    unix`sys_syscall+0x17a
    1

    unix`splr+0x92
    genunix`thread_lock+0x1d
    genunix`yield+0x23
    unix`_sys_sysenter_post_swapgs+0x149
    1

    unix`splr+0x92
    unix`lock_set_spl+0x1d
    genunix`disp_lock_enter+0x2e
    unix`disp+0xad
    unix`swtch+0xa4
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    1

    genunix`bt_getlowbit+0x45
    poll`dp_pcache_poll+0x104
    poll`dpioctl+0x2dd
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    1

    genunix`cpu_update_pct+0x45
    unix`setbackdq+0x299
    FSS`fss_wakeup+0x94
    genunix`sleepq_wakeone_chan+0x89
    genunix`cv_signal+0x8e
    genunix`lwp_unpark+0x62
    genunix`syslwp_park+0x42
    unix`sys_syscall+0x17a
    1

    unix`page_vpadd+0x36
    unix`page_do_hashin+0x11a
    unix`page_hashin+0xd4
    unix`page_create_va+0x3b7
    genunix`swap_getapage+0xd9
    genunix`swap_getpage+0x85
    genunix`fop_getpage+0x9a
    genunix`anon_zero+0xa3
    genunix`segvn_faultpage+0x2a4
    genunix`segvn_fault+0xc13
    genunix`as_fault+0x5ee
    unix`pagefault+0x99
    unix`trap+0xe63
    unix`0xfffffffffb8001d6
    1

    genunix`bt_getlowbit+0x64
    poll`dp_pcache_poll+0x104
    poll`dpioctl+0x2dd
    genunix`cdev_ioctl+0x45
    specfs`spec_ioctl+0x5a
    genunix`fop_ioctl+0x7b
    genunix`ioctl+0x18e
    unix`sys_syscall+0x17a
    2

    unix`tsc_read+0x3
    genunix`gethrtime_unscaled+0xd
    genunix`syscall_mstate+0x5c
    unix`sys_syscall+0x10e
    2

    unix`do_splx+0x8d
    unix`swtch+0x184
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    2

    genunix`sleepq_wakeone_chan+0x28
    genunix`cv_signal+0x8e
    mac`mac_rx_srs_process+0x1b4
    mac`mac_tx_send+0x334
    mac`mac_tx_single_ring_mode+0x91
    mac`mac_tx+0x35d
    dld`str_mdata_fastpath_put+0xa4
    ip`ip_xmit+0x7eb
    ip`ire_send_wire_v4+0x345
    ip`conn_ip_output+0x250
    ip`tcp_output+0x86d
    ip`squeue_enter+0x440
    ip`tcp_sendmsg+0x104
    sockfs`so_sendmsg+0x21b
    sockfs`socket_sendmsg+0x61
    sockfs`socket_vop_write+0x63
    genunix`fop_write+0xd3
    genunix`writev+0x41a
    unix`sys_syscall+0x17a
    2

    unix`lock_set+0x4
    unix`caps_charge_adjust+0x6a
    unix`cpucaps_charge+0x61
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    2

    unix`atomic_add_64+0x4
    unix`sys_syscall+0x1a1
    2

    unix`tsc_read+0x3
    genunix`gethrtime_unscaled+0xd
    genunix`syscall_mstate+0x5c
    unix`sys_syscall+0x1a1
    3

    unix`bitset_in_set+0x3c
    unix`cpu_wakeup_mwait+0x36
    unix`setbackdq+0x468
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    3

    unix`lock_set+0x4
    unix`setbackdq+0x2c2
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    3

    unix`cpucaps_charge+0x53
    FSS`fss_yield+0x34
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    4

    unix`splr+0x92
    genunix`thread_lock+0x1d
    genunix`yield+0x23
    unix`sys_syscall+0x17a
    4

    unix`sys_syscall+0xff
    5

    unix`membar_enter+0x3
    unix`disp+0x198
    unix`swtch+0xa4
    genunix`yield+0x57
    unix`sys_syscall+0x17a
    5

    unix`membar_enter+0x3
    FSS`fss_yield+0xb2
    genunix`yield+0x46
    unix`sys_syscall+0x17a
    5

    unix`do_splx+0x8d
    genunix`disp_lock_exit_nopreempt+0x43
    genunix`yield+0x52
    unix`sys_syscall+0x17a
    6

    2691
    [root@headnode (bh1-kvm1:0) ~]#