使用crash跟踪linux内核（一）-crash安装

一、 查看操作系统版本

[root@kongu 桌面]# uname -a

Linux kongu 2.6.32-431.el6.x86_64 #1 SMP Sun Nov 10 22:19:54 EST 2013 x86_64 x86_64 x86_64 GNU/Linux

二、 crash安装

RedHat 6.5安装光盘带有crash安装包，直接安装即可。也可以下载安装，地址：http://people.redhat.com/anderson/

[root@kongu Packages]# crash -v

crash 6.1.0-5.el6

This program is free software, covered by the GNU General Public License,

and you are welcome to change it and/or distribute copies of it under

certain conditions. Enter "help copying" to see the conditions.

This program has absolutely no warranty. Enter "help warranty" for details.

GNU gdb (GDB) 7.3.1

License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>

This is free software: you are free to change and redistribute it.

There is NO WARRANTY, to the extent permitted by law. Type "show copying"

and "show warranty" for details.

This GDB was configured as "x86_64-unknown-linux-gnu".

三、 下载相应内核版本的调试信息包，下载安装包如下 ：

1、kernel-debuginfo-common-x86_64-2.6.32-431.el6.x86_64.rpm

2、kernel-debuginfo-2.6.32-431.el6.x86_64.rpm

下载地址：http://debuginfo.centos.org/6/x86_64/

四、 安装内核调试信息包

[root@kongu 下载]# rpm -ivh kernel-debuginfo-common-x86_64-2.6.32-431.el6.x86_64.rpm

warning: kernel-debuginfo-common-x86_64-2.6.32-431.el6.x86_64.rpm: Header V3 RSA/SHA1 Signature, key ID c105b9de: NOKEY

Preparing... ########################################### [100%]

1:kernel-debuginfo-common########################################### [100%]

[root@kongu 下载]# rpm -ihv kernel-debuginfo-2.6.32-431.el6.x86_64.rpm

warning: kernel-debuginfo-2.6.32-431.el6.x86_64.rpm: Header V3 RSA/SHA1 Signature, key ID c105b9de: NOKEY

Preparing... ########################################### [100%]

1:kernel-debuginfo ########################################### [100%]

[root@kongu 下载]#

注意：安装包存在依赖关系，需要先安装kernel-debuginfo-common-x86_64-2.6.32-431.el6.x86_64.rpm

五、 跟踪内核

安装调试信息后，在/usr/lib/debug/lib/modules/2.6.32-431.el6.x86_64/目录下，生产带调试信息内核映像vmlinux。System.map-2.6.32-431.el6.x86_64为对应内核符号信息。

[root@kongu usr]# crash /usr/lib/debug/lib/modules/2.6.32-431.el6.x86_64/vmlinux /boot/System.map-2.6.32-431.el6.x86_64

crash 6.1.0-5.el6

This program is free software, covered by the GNU General Public License,

and you are welcome to change it and/or distribute copies of it under

certain conditions. Enter "help copying" to see the conditions.

This program has absolutely no warranty. Enter "help warranty" for details.

GNU gdb (GDB) 7.3.1

License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>

This is free software: you are free to change and redistribute it.

There is NO WARRANTY, to the extent permitted by law. Type "show copying"

and "show warranty" for details.

This GDB was configured as "x86_64-unknown-linux-gnu"...

SYSTEM MAP: /boot/System.map-2.6.32-431.el6.x86_64

DEBUG KERNEL: /usr/lib/debug/lib/modules/2.6.32-431.el6.x86_64/vmlinux (2.6.32-431.el6.x86_64)

DUMPFILE: /dev/crash

CPUS: 2

DATE: Mon Jul 10 20:23:33 2017

UPTIME: 01:11:34

LOAD AVERAGE: 0.40, 0.34, 0.20

TASKS: 288

NODENAME: kongu

RELEASE: 2.6.32-431.el6.x86_64

VERSION: #1 SMP Sun Nov 10 22:19:54 EST 2013

MACHINE: x86_64 (2294 Mhz)

MEMORY: 2 GB

PID: 13183

COMMAND: "crash"

TASK: ffff880012454aa0 [THREAD_INFO: ffff88007c0d0000]

CPU: 1

STATE: TASK_RUNNING (ACTIVE)

crash> !ps -ef | grep sshd //查看进程sshd的进程ID

root 1909 1 0 19:12 ? 00:00:00 /usr/sbin/sshd

root 14504 13183 0 20:32 pts/3 00:00:00 sh -c ps -ef | grep sshd

root 14506 14504 0 20:32 pts/3 00:00:00 grep sshd

crash> task 1909 //设置sshd为当前跟踪进程

PID: 1909 TASK: ffff880037ba3540 CPU: 0 COMMAND: "sshd"

struct task_struct {

state = 1,

stack = 0xffff880078658000,

usage = {

counter = 2

flags = 4202816,

ptrace = 0,

lock_depth = -1,

prio = 120,

static_prio = 120,

normal_prio = 120,

rt_priority = 0,

sched_class = 0xffffffff8160be60,

se = {

load = {

weight = 1024,

inv_weight = 4194304

run_node = {

rb_parent_color = 1,

rb_right = 0x0,

rb_left = 0x0

group_node = {

next = 0xffff880037ba35a0,

prev = 0xffff880037ba35a0

on_rq = 0,

exec_start = 22563864022,

sum_exec_runtime = 5929509,

vruntime = 9280442114,

prev_sum_exec_runtime = 5696312,

last_wakeup = 0,

avg_overlap = 0,

nr_migrations = 1,

start_runtime = 0,

avg_wakeup = 0,

avg_running = 0,

wait_start = 0,

wait_max = 1215452,

wait_count = 3,

wait_sum = 1234768,

iowait_count = 78,

iowait_sum = 468265912,

sleep_start = 22563858600,

sleep_max = 0,

sum_sleep_runtime = 0,

block_start = 0,

block_max = 0,

exec_max = 1026762,

slice_max = 0,

nr_migrations_cold = 0,

nr_failed_migrations_affine = 0,

nr_failed_migrations_running = 0,

nr_failed_migrations_hot = 0,

nr_forced_migrations = 0,

nr_forced2_migrations = 0,

nr_wakeups = 0,

nr_wakeups_sync = 0,

nr_wakeups_migrate = 0,

nr_wakeups_local = 0,

nr_wakeups_remote = 0,

nr_wakeups_affine = 0,

nr_wakeups_affine_attempts = 0,

nr_wakeups_passive = 0,

nr_wakeups_idle = 0,

parent = 0x0,

cfs_rq = 0xffff88000c4168a8,

my_q = 0x0,

rh_reserved = 0

rt = {

run_list = {

next = 0xffff880037ba3708,

prev = 0xffff880037ba3708

timeout = 0,

time_slice = 100,

nr_cpus_allowed = 64,

back = 0x0,

parent = 0x0,

rt_rq = 0xffff88000c4169c8,

my_q = 0x0

preempt_notifiers = {

first = 0x0

fpu_counter = 3 '\003',

btrace_seq = 0,

policy = 0,

cpus_allowed = {

bits = {18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615}

sched_info = {

pcount = 3,

run_delay = 1234768,

last_arrival = 22563630825,

last_queued = 0,

bkl_count = 0

tasks = {

next = 0xffff88007da52f28,

prev = 0xffff880037a23948

pushable_tasks = {

prio = 140,

plist = {

prio_list = {

next = 0xffff880037ba39a0,

prev = 0xffff880037ba39a0

node_list = {

next = 0xffff880037ba39b0,

prev = 0xffff880037ba39b0

}

mm = 0xffff880037ec7340,

active_mm = 0xffff880037ec7340,

exit_state = 0,

exit_code = 0,

exit_signal = 17,

pdeath_signal = 0,

personality = 0,

did_exec = 0,

in_execve = 0,

in_iowait = 0,

sched_reset_on_fork = 0,

pid = 1909,

tgid = 1909,

stack_canary = 3329918480,

real_parent = 0xffff88007e531500,

parent = 0xffff88007e531500,

children = {

next = 0xffff880037ba3a08,

prev = 0xffff880037ba3a08

sibling = {

next = 0xffff88007da52fb8,

prev = 0xffff880037a239d8

group_leader = 0xffff880037ba3540,

ptraced = {

next = 0xffff880037ba3a30,

prev = 0xffff880037ba3a30

ptrace_entry = {

next = 0xffff880037ba3a40,

prev = 0xffff880037ba3a40

pids = {{

node = {

next = 0x0,

pprev = 0xffff880037f1ca48

pid = 0xffff880037f1ca40

}, {

node = {

next = 0x0,

pprev = 0xffff880037f1ca50

pid = 0xffff880037f1ca40

}, {

node = {

next = 0x0,

pprev = 0xffff880037f1ca58

pid = 0xffff880037f1ca40

}},

thread_group = {

next = 0xffff880037ba3a98,

prev = 0xffff880037ba3a98

vfork_done = 0x0,

set_child_tid = 0x7ffc736e5a90,

clear_child_tid = 0x7ffc736e5a90,

utime = 0,

stime = 6,

utimescaled = 0,

stimescaled = 6,

gtime = 0,

prev_utime = 0,

prev_stime = 0,

nvcsw = 1,

nivcsw = 2,

start_time = {

tv_sec = 22,

tv_nsec = 409675660

real_start_time = {

tv_sec = 22,

tv_nsec = 409675660

min_flt = 167,

maj_flt = 0,

cputime_expires = {

utime = 0,

stime = 0,

sum_exec_runtime = 0

cpu_timers = {{

next = 0xffff880037ba3b50,

prev = 0xffff880037ba3b50

}, {

next = 0xffff880037ba3b60,

prev = 0xffff880037ba3b60

}, {

next = 0xffff880037ba3b70,

prev = 0xffff880037ba3b70

}},

real_cred = 0xffff88007a0ae8c0,

cred = 0xffff88007a0ae8c0,

cred_guard_mutex = {

count = {

counter = 1

wait_lock = {

raw_lock = {

slock = 0

}

wait_list = {

next = 0xffff880037ba3b98,

prev = 0xffff880037ba3b98

owner = 0x0

replacement_session_keyring = 0x0,

comm = "sshd\000\000\000\000\000\000\000\000\000\000\000", //进程名

link_count = 0,

total_link_count = 0,

sysvsem = {

undo_list = 0x0

last_switch_count = 0,

thread = {

tls_array = {{

{

a = 0,

b = 0

{

limit0 = 0,

base0 = 0,

base1 = 0,

type = 0,

s = 0,

dpl = 0,

p = 0,

limit = 0,

avl = 0,

l = 0,

d = 0,

g = 0,

base2 = 0

}

}, {

{

a = 0,

b = 0

{

limit0 = 0,

base0 = 0,

base1 = 0,

type = 0,

s = 0,

dpl = 0,

p = 0,

limit = 0,

avl = 0,

l = 0,

d = 0,

g = 0,

base2 = 0

}

}, {

{

a = 0,

b = 0

{

limit0 = 0,

base0 = 0,

base1 = 0,

type = 0,

s = 0,

dpl = 0,

p = 0,

limit = 0,

avl = 0,

l = 0,

d = 0,

g = 0,

base2 = 0

}

}},

sp0 = 18446612134334144512,

sp = 18446612134334142536,

usersp = 140736137125640,

es = 0,

ds = 0,

fsindex = 0,

gsindex = 0,

fs = 140722245097408,

gs = 0,

debugreg0 = 0,

debugreg1 = 0,

debugreg2 = 0,

debugreg3 = 0,

debugreg6 = 0,

debugreg7 = 0,

cr2 = 0,

trap_no = 0,

error_code = 0,

xstate = 0xffff880037b7d140,

io_bitmap_ptr = 0x0,

iopl = 0,

io_bitmap_max = 0

fs = 0xffff88007e55ef00,

files = 0xffff880037b75740,

nsproxy = 0xffffffff81aa5840,

signal = 0xffff880037babac0,

sighand = 0xffff880037b59600,

blocked = {

sig = {0}

real_blocked = {

sig = {0}

saved_sigmask = {

sig = {0}

pending = {

list = {

next = 0xffff880037ba3cd0,

prev = 0xffff880037ba3cd0

signal = {

sig = {0}

}

sas_ss_sp = 0,

sas_ss_size = 0,

notifier = 0,

notifier_data = 0x0,

notifier_mask = 0x0,

audit_context = 0xffff880037cce800,

loginuid = 4294967295,

sessionid = 4294967295,

seccomp = {<No data fields>},

utrace = 0x0,

utrace_flags = 0,

parent_exec_id = 7,

self_exec_id = 7,

alloc_lock = {

raw_lock = {

slock = 0

}

irqaction = 0x0,

pi_lock = {

raw_lock = {

slock = 0

}

pi_waiters = {

prio_list = {

next = 0xffff880037ba3d50,

prev = 0xffff880037ba3d50

node_list = {

next = 0xffff880037ba3d60,

prev = 0xffff880037ba3d60

}

pi_blocked_on = 0x0,

journal_info = 0x0,

bio_list = 0x0,

bio_tail = 0x0,

reclaim_state = 0x0,

backing_dev_info = 0x0,

io_context = 0x0,

ptrace_message = 0,

last_siginfo = 0x0,

ioac = {

rchar = 4623,

wchar = 11,

syscr = 3,

syscw = 2,

read_bytes = 0,

write_bytes = 4096,

cancelled_write_bytes = 0

acct_rss_mem1 = 1584414,

acct_vm_mem1 = 96167736,

acct_timexpd = 6,

mems_allowed = {

bits = {1, 0, 0, 0, 0, 0, 0, 0}

cpuset_mem_spread_rotor = 0,

cpuset_slab_spread_rotor = 0,

mems_allowed_change_disable = 0,

cgroups = 0xffffffff81eda460,

cg_list = {

next = 0xffff880037ba3e58,

prev = 0xffff880037ba3e58

robust_list = 0x7ffc736e5aa0,

compat_robust_list = 0x0,

pi_state_list = {

next = 0xffff880037ba3e78,

prev = 0xffff880037ba3e78

pi_state_cache = 0x0,

__reserved_perf__ = 0x0,

perf_event_mutex = {

count = {

counter = 1

wait_lock = {

raw_lock = {

slock = 0

}

wait_list = {

next = 0xffff880037ba3ea0,

prev = 0xffff880037ba3ea0

owner = 0x0

perf_event_list = {

next = 0xffff880037ba3eb8,

prev = 0xffff880037ba3eb8

mempolicy = 0x0,

il_next = 0,

fs_excl = {

counter = 0

rcu = {

next = 0x0,

func = 0

splice_pipe = 0x0,

delays = 0xffff8800379aec88,

dirties = {

events = 1,

period = 0,

shift = 0,

lock = {

raw_lock = {

slock = 0

}

timer_slack_ns = 50000,

default_timer_slack_ns = 50000,

scm_work_list = 0x0,

curr_ret_stack = -1,

ret_stack = 0x0,

ftrace_timestamp = 0,

trace_overrun = {

counter = 0

tracing_graph_pause = {

counter = 0

trace = 0,

trace_recursion = 0,

rh_reserved = {0, 0},

perf_event_ctxp = {0x0, 0x0},

memcg_batch = {

do_batch = 0,

memcg = 0x0,

bytes = 0,

memsw_bytes = 0

}

六、问题

1、使用/proc/kcore,/dev/mem启动跟踪内核实时状态，启动都不成功，不知为何！

[root@kongu usr]# crash /usr/lib/debug/lib/modules/2.6.32-431.el6.x86_64/vmlinux /proc/kcore

crash 6.1.0-5.el6
Copyright (C) 2002-2012 Red Hat, Inc.
Copyright (C) 2004, 2005, 2006, 2010 IBM Corporation
Copyright (C) 1999-2006 Hewlett-Packard Co
Copyright (C) 2005, 2006, 2011, 2012 Fujitsu Limited
Copyright (C) 2006, 2007 VA Linux Systems Japan K.K.
Copyright (C) 2005, 2011 NEC Corporation
Copyright (C) 1999, 2002, 2007 Silicon Graphics, Inc.
Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc.
This program is free software, covered by the GNU General Public License,
and you are welcome to change it and/or distribute copies of it under
certain conditions. Enter "help copying" to see the conditions.
This program has absolutely no warranty. Enter "help warranty" for details.

/proc/kcore: read: No such file or directory

crash: /proc/kcore: initialization failed

[root@kongu usr]# crash /usr/lib/debug/lib/modules/2.6.32-431.el6.x86_64/vmlinux /dev/mem

crash 6.1.0-5.el6
Copyright (C) 2002-2012 Red Hat, Inc.
Copyright (C) 2004, 2005, 2006, 2010 IBM Corporation
Copyright (C) 1999-2006 Hewlett-Packard Co
Copyright (C) 2005, 2006, 2011, 2012 Fujitsu Limited
Copyright (C) 2006, 2007 VA Linux Systems Japan K.K.
Copyright (C) 2005, 2011 NEC Corporation
Copyright (C) 1999, 2002, 2007 Silicon Graphics, Inc.
Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc.
This program is free software, covered by the GNU General Public License,
and you are welcome to change it and/or distribute copies of it under
certain conditions. Enter "help copying" to see the conditions.
This program has absolutely no warranty. Enter "help warranty" for details.

WARNING: /usr/lib/debug/lib/modules/2.6.32-431.el6.x86_64/vmlinux
and /proc/version do not match!

WARNING: /proc/version indicates kernel version: 2.6.32-431.el6.x86_64

crash: please use the vmlinux file for that kernel version, or try using
the System.map for that kernel version as an additional argument.

[root@kongu usr]#

posted @ 2017-07-11 11:45 已秋阅读(119) 评论(0) 收藏举报

刷新页面返回顶部

已秋

使用crash跟踪linux内核（一）-crash安装

公告