一、如何确定探测点
这就需要跟踪一下sshd进程的执行流程，分析一下密码输入的上下文执行函数的位置。1#函数还真的挺多- - 2# stap -l 'process("/usr/sbin/sshd").function("*")' | wc -l 31433
二、编写SystemTap安装探测点 ：
1.探测点思路（ 信号处理方式）：
在指定内核信号文件中匹配所有函数并打上探测点 ，分别打上 call和return两个探测点 ， call和retrun的时候输出函数名 ， 并利用thread_indent函数增加缩进， 这样就可以体现出函数的调用过程了。因为内核处理信号比较频繁，所以上面脚本中就用target()来过滤指定进程pid值（stap -x），只要一个pid的信号处理流程，这样输出比较少才好分析。
1# stap -l 'kernel.function("sys_signal")' 2kernel.function("SyS_signal@kernel/signal.c:3554") 3#pid=1710 4# ps -ef | grep '/usr/sbin/sshd' |grep -v 'grep' 5root 1710 1 0 Aug31 ? 00:00:00 /usr/sbin/sshd -D2.内核跟踪代码的执行流程Tap：
1# cat kernel_signal_process.stp 2probe begin { 3 printf("Tracert begin...\n") 4} 5 6#function::thread_indent — returns an amount of space with the current task information 7#function::ppfunc — Returns the function name parsed from pp 8#function::target — Return the process ID of the target process 9probe kernel.function("*@kernel/signal.c").call { 10 if (target() == pid()) { 11 printf("%s -> %s\n", thread_indent(4),ppfunc()) 12 } 13 } 14 15probe kernel.function("*@kernel/signal.c").return { 16 if (target() == pid()) { 17 printf("%s <- %s\n", thread_indent(-4), ppfunc()) 18 } 19}3. 启动SystemTap安装探测点
1# stap -x 1710 kernel_signal_process.stp 2WARNING: function signals_init is in blacklisted section: keyword at kernel_signal_process.stp:8:1 3 source: probe kernel.function("*@kernel/signal.c").call { 4 ^ 5WARNING: function setup_print_fatal_signals is in blacklisted section: keyword at :8:1 6 source: probe kernel.function("*@kernel/signal.c").call { 7 ^ 8Tracert begin... 9 0 sshd(1710): -> __lock_task_sighand 10 9 sshd(1710): <- __lock_task_sighand 11 0 sshd(1710): -> recalc_sigpending 12 5 sshd(1710): -> recalc_sigpending_tsk 13 8 sshd(1710): <- recalc_sigpending_tsk 14 10 sshd(1710): <- recalc_sigpending 15#输入密码之后的执行流程 16 10 sshd(1710): <- recalc_sigpending 17 0 sshd(1710): -> do_notify_resume 18 10 sshd(1710): <- do_notify_resume 19#退出登录的执行流程 20 0 sshd(1710): -> do_notify_resume 21 8 sshd(1710): -> do_signal 22 10 sshd(1710): -> get_signal_to_deliver 23 13 sshd(1710): -> dequeue_signal 24 15 sshd(1710): -> __dequeue_signal 25 18 sshd(1710): <- __dequeue_signal 26 20 sshd(1710): -> __dequeue_signal 27 23 sshd(1710): -> __sigqueue_free 28 26 sshd(1710): <- __sigqueue_free 29 27 sshd(1710): <- __dequeue_signal 30 29 sshd(1710): -> recalc_sigpending 31 31 sshd(1710): -> recalc_sigpending_tsk 32 44 sshd(1710): <- recalc_sigpending_tsk 33 46 sshd(1710): <- recalc_sigpending 34 48 sshd(1710): <- dequeue_signal 35 50 sshd(1710): <- get_signal_to_deliver 36 53 sshd(1710): -> setup_sigcontext 37 55 sshd(1710): <- setup_sigcontext 38 57 sshd(1710): -> signal_setup_done 39 60 sshd(1710): -> signal_delivered 40 62 sshd(1710): -> __set_current_blocked 41 65 sshd(1710): -> __set_task_blocked 42 67 sshd(1710): -> recalc_sigpending 43 68 sshd(1710): -> recalc_sigpending_tsk 44 70 sshd(1710): <- recalc_sigpending_tsk 45 72 sshd(1710): <- recalc_sigpending 46 73 sshd(1710): <- __set_task_blocked 47 75 sshd(1710): <- __set_current_blocked 48 77 sshd(1710): <- signal_delivered 49 78 sshd(1710): <- signal_setup_done 50 80 sshd(1710): <- do_signal 51 82 sshd(1710): <- do_notify_resume 52 0 sshd(1710): -> flush_sigqueue 53 2 sshd(1710): <- flush_sigqueue 54 0 sshd(1710): -> flush_sigqueue 55 1 sshd(1710): <- flush_sigqueue 56 0 sshd(1710): -> SyS_rt_sigaction 57 2 sshd(1710): -> do_sigaction 58 4 sshd(1710): <- do_sigaction 59 5 sshd(1710): <- SyS_rt_sigaction 60 0 sshd(1710): -> sys_rt_sigreturn 61 2 sshd(1710): -> set_current_blocked 62 4 sshd(1710): -> __set_current_blocked 63 6 sshd(1710): -> __set_task_blocked 64 7 sshd(1710): -> recalc_sigpending 65 9 sshd(1710): -> recalc_sigpending_tsk 66 11 sshd(1710): <- recalc_sigpending_tsk 67 12 sshd(1710): <- recalc_sigpending 68 14 sshd(1710): <- __set_task_blocked 69 15 sshd(1710): <- __set_current_blocked 70 17 sshd(1710): <- set_current_blocked 71 19 sshd(1710): -> restore_sigcontext 72 21 sshd(1710): <- restore_sigcontext 73 22 sshd(1710): -> restore_altstack 74 24 sshd(1710): -> do_sigaltstack 75 26 sshd(1710): <- do_sigaltstack 76 28 sshd(1710): <- restore_altstack 77 29 sshd(1710): <- sys_rt_sigreturn 78 79 80# stap -l 'kernel.function("recalc_sigpending")' 81kernel.function("recalc_sigpending@kernel/signal.c:160") 82#内核空间和用户空间交换数据时，使用get_user*和put_user*函数完成简单类型变量拷贝任务 83158 #ifdef CONFIG_X86_32 84159 put_user_ex(regs->cs, (unsigned int __user *)&sc->cs); 85160 put_user_ex(regs->flags, &sc->flags); 86161 put_user_ex(regs->sp, &sc->sp_at_signal); 87162 put_user_ex(regs->ss, (unsigned int __user *)&sc->ss);4.使用strace命令能更清晰的看到程序执行的调用流程：
1#trace=signal 2#strace -tt -f -e trace=signal -p 1710 3strace: Process 1710 attached 4strace: Process 39362 attached 5[pid 39362] 23:28:57.980098 rt_sigaction(SIGRTMIN, {sa_handler=0x7fea2694b860, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7fea26954630}, NULL, 8) = 0 6[pid 39362] 23:28:57.980326 rt_sigaction(SIGRT_1, {sa_handler=0x7fea2694b8f0, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7fea26954630}, NULL, 8) = 0 7[pid 39362] 23:28:57.980391 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0 8[pid 39362] 23:28:57.988670 rt_sigaction(SIGPIPE, NULL, {sa_handler=SIG_IGN, sa_mask=[], sa_flags=0}, 8) = 0 9[pid 39362] 23:28:57.989008 rt_sigaction(SIGALRM, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 10[pid 39362] 23:28:57.989067 rt_sigaction(SIGHUP, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 11[pid 39362] 23:28:57.989120 rt_sigaction(SIGTERM, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 12[pid 39362] 23:28:57.989172 rt_sigaction(SIGQUIT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 13[pid 39362] 23:28:57.989220 rt_sigaction(SIGCHLD, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 14[pid 39362] 23:28:57.989268 rt_sigaction(SIGINT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 15[pid 39362] 23:28:57.989954 rt_sigaction(SIGALRM, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 16[pid 39362] 23:28:57.990029 rt_sigaction(SIGALRM, {sa_handler=0x562069f8bf30, sa_mask=[], sa_flags=SA_RESTORER|SA_INTERRUPT, sa_restorer=0x7fea28650400}, NULL, 8) = 0 17strace: Process 39363 attached 18 19#trace=file 20# strace -tt -f -e trace=file -p 1710 21strace: Process 1710 attached 22[pid 40200] 23:33:45.287284 open("/proc/self/oom_score_adj", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 10 23[pid 40200] 23:33:45.288205 execve("/usr/sbin/sshd", ["/usr/sbin/sshd", "-D", "-R"], 0x564a53c46ab0 /* 4 vars */) = 0 24[pid 40200] 23:33:45.288958 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) 25[pid 40200] 23:33:45.289206 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 26[pid 40200] 23:33:45.289367 open("/lib64/libfipscheck.so.1", O_RDONLY|O_CLOEXEC) = 3 27[pid 40200] 23:33:45.289642 open("/lib64/libwrap.so.0", O_RDONLY|O_CLOEXEC) = 3 28... 29[pid 40200] 23:33:45.374997 open("/etc/passwd", O_RDONLY|O_CLOEXEC) = 4 30[pid 40200] 23:33:45.375237 stat("/bin/bash", {st_mode=S_IFREG|0755, st_size=964536, ...}) = 0 31[pid 40200] 23:33:45.375697 stat("/etc/pam.d", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0 32[pid 40200] 23:33:45.375770 open("/etc/pam.d/sshd", O_RDONLY) = 4 33[pid 40200] 23:33:45.376006 open("/usr/lib64/security/pam_sepermit.so", O_RDONLY|O_CLOEXEC) = 8 34[pid 40200] 23:33:45.376378 open("/etc/pam.d/password-auth", O_RDONLY) = 8 35[pid 40200] 23:33:45.376553 open("/usr/lib64/security/pam_env.so", O_RDONLY|O_CLOEXEC) = 9 36[pid 40200] 23:33:45.376899 open("/usr/lib64/security/pam_faildelay.so", O_RDONLY|O_CLOEXEC) = 9 37[pid 40200] 23:33:45.377230 open("/usr/lib64/security/pam_unix.so", O_RDONLY|O_CLOEXEC) = 9 38[pid 40200] 23:33:45.377694 open("/usr/lib64/security/pam_succeed_if.so", O_RDONLY|O_CLOEXEC) = 9 39[pid 40200] 23:33:45.377981 stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7fff90575f10) = -1 ENOENT (No such file or directory) 40[pid 40200] 23:33:45.378098 open("/usr/lib64/security/pam_deny.so", O_RDONLY|O_CLOEXEC) = 9 41[pid 40200] 23:33:45.378513 open("/etc/pam.d/postlogin", O_RDONLY) = 8 42[pid 40200] 23:33:45.378769 open("/usr/lib64/security/pam_reauthorize.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) 43[pid 40200] 23:33:45.378834 stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7fff905765f0) = -1 ENOENT (No such file or directory) 44[pid 40200] 23:33:45.378913 open("/usr/lib64/security/pam_nologin.so", O_RDONLY|O_CLOEXEC) = 8 45[pid 40200] 23:33:45.379227 open("/etc/pam.d/password-auth", O_RDONLY) = 8 46[pid 40200] 23:33:45.379396 open("/usr/lib64/security/pam_localuser.so", O_RDONLY|O_CLOEXEC) = 9 47[pid 40200] 23:33:45.379723 open("/usr/lib64/security/pam_permit.so", O_RDONLY|O_CLOEXEC) = 9 48[pid 40200] 23:33:45.380115 open("/etc/pam.d/password-auth", O_RDONLY) = 8 49[pid 40200] 23:33:45.380283 open("/usr/lib64/security/pam_pwquality.so", O_RDONLY|O_CLOEXEC) = 9 50[pid 40200] 23:33:45.380556 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 9 51[pid 40200] 23:33:45.380716 open("/lib64/libpwquality.so.1", O_RDONLY|O_CLOEXEC) = 9 52[pid 40200] 23:33:45.380982 open("/lib64/libcrack.so.2", O_RDONLY|O_CLOEXEC) = 9 53[pid 40200] 23:33:45.381391 stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7fff90575f00) = -1 ENOENT (No such file or directory) 54[pid 40200] 23:33:45.381716 open("/usr/lib64/security/pam_selinux.so", O_RDONLY|O_CLOEXEC) = 8 55[pid 40200] 23:33:45.382076 open("/usr/lib64/security/pam_loginuid.so", O_RDONLY|O_CLOEXEC) = 8 56[pid 40200] 23:33:45.382411 open("/usr/lib64/security/pam_namespace.so", O_RDONLY|O_CLOEXEC) = 8 57[pid 40200] 23:33:45.382752 stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7fff90576470) = -1 ENOENT (No such file or directory) 58[pid 40200] 23:33:45.382874 open("/usr/lib64/security/pam_keyinit.so", O_RDONLY|O_CLOEXEC) = 8 59[pid 40200] 23:33:45.383203 open("/etc/pam.d/password-auth", O_RDONLY) = 8 60[pid 40200] 23:33:45.383397 open("/usr/lib64/security/pam_limits.so", O_RDONLY|O_CLOEXEC) = 9 61[pid 40200] 23:33:45.383760 open("/usr/lib64/security/pam_systemd.so", O_RDONLY|O_CLOEXEC) = 9 62[pid 40200] 23:33:45.384034 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 9 63[pid 40200] 23:33:45.384196 open("/lib64/libpam_misc.so.0", O_RDONLY|O_CLOEXEC) = 9 64[pid 40200] 23:33:45.384928 open("/etc/pam.d/postlogin", O_RDONLY) = 8 65[pid 40200] 23:33:45.385115 open("/usr/lib64/security/pam_lastlog.so", O_RDONLY|O_CLOEXEC) = 9 66[pid 40200] 23:33:45.385677 open("/etc/pam.d/other", O_RDONLY) = 4 67.... 68 69#其实可以看到在PAM认证时相关的文件，在挂载对应的函数point.5.使用process 跟踪调用的函数执行流程：
1# cat func_trace.stp 2probe process("/usr/sbin/sshd").function("*").call 3{ 4 printf("%s -> %s\n", thread_indent(4), ppfunc()); 5} 6 7probe process("/usr/sbin/sshd").function("*").return 8{ 9 printf("%s <- %s\n", thread_indent(-4), ppfunc()); 10} 11 12###################################################################### 139233994 sshd(14355): <- buffer_get_string 149233998 sshd(14355): -> auth_password 159234007 sshd(14355): -> sshpam_auth_passwd 169234352 sshd(14355): -> sshpam_passwd_conv 179234358 sshd(14355): -> debug3 189234363 sshd(14355): -> do_log 199234368 sshd(14355): <- do_log 209234370 sshd(14355): <- debug3 219234374 sshd(14355): <- sshpam_passwd_conv 2211649789 sshd(14355): -> debug 2311649804 sshd(14355): -> do_log 2411649810 sshd(14355): <- do_log 2511649813 sshd(14355): <- debug 2611649817 sshd(14355): <- sshpam_auth_passwd 2711649823 sshd(14355): <- auth_password 2811649829 sshd(14355): -> explicit_bzero 2911649834 sshd(14355): <- explicit_bzero 3011649840 sshd(14355): -> sshbuf_reset 3111649845 sshd(14355): -> explicit_bzero 3211649849 sshd(14355): <- explicit_bzero 3311649852 sshd(14355): <- sshbuf_reset 3411649857 sshd(14355): -> buffer_put_int 3511649863 sshd(14355): -> sshbuf_put_u32 3611649868 sshd(14355): -> sshbuf_reserve 3711649873 sshd(14355): -> sshbuf_allocate 3811649879 sshd(14355): -> sshbuf_check_reserve 3911649883 sshd(14355): <- sshbuf_check_reserve 4011649888 sshd(14355): -> sshbuf_maybe_pack 4111649893 sshd(14355): <- sshbuf_maybe_pack 4211649895 sshd(14355): <- sshbuf_allocate