Re: Docs on multi-threaded debugging using gdbserver?

[Grant Edwards]

On 2009-06-19, Grant Edwards <address@hidden> wrote:

> Is there any documentation on how to set up for remote
> multi-threaded debugging using gdb-server?

[...]

> Here's a typical session:
>
>     GNU gdb 6.8

[target remote ....]

[...]
>     0x40000930 in _start () from 
> /home/grante/processors/atmel/RM9200/buildroot-2009.05/project_build_arm/uclibc/root/lib/ld-uClibc.so.0
>     (gdb) break main
>     Breakpoint 1 at 0x8bf4: file hello.c, line 45.
>     (gdb) c
>     Continuing.
>     
>     Breakpoint 1, main () at hello.c:45
>     45        pthread_attr_t attr, *attrp=NULL;
>     (gdb) next
[...]
>     62            s = pthread_create(tid+i, attrp, thread, (void*)i);  
> abort_if_error(s);
>     (gdb) next
>     
>     Program received signal SIG32, Real-time event 32.
>     0x400443d4 in ?? ()
>     (gdb) 
>     Cannot find bounds of current function

Using strace I've verified that

 1) The dev-host gdb is finding the target-arch .so files
    correctly (I think that can also be deduced from the line
    above that says "in _start () from ..../ld-uClibc.so.0")

 2) The target gdbserver is finding libthread_db.so.

I've also confirmed that the target arch libpthread .so files
are not stripped (either on the dev-host or on the target).

I've tried using both a stripped and unstripped application
binary on the target.  [I always use an unstripped ELF
executable that still has debug symbols on the dev-host as the
target for gdb's "file" command.]

I've got strace output for both gdb and gdbserver, and it
always shows gdbserver responding to a $vCont command with $T4d
immediately before gdb prints the message about SIG32.  The
$vCont command in question was the one that was issued as a
result of the "next" command that was stepping through the call
to pthread_create() shown at source file line 62 above:

  ------------------------------gdb strace------------------------------
  send(5, "$vCont;c#a8"..., 11, 0)        = 11
  select(6, [5], NULL, [5], {1, 0})       = 1 (in [5], left {1, 0})
  recv(5, "+"..., 8192, 0)                = 1
  rt_sigaction(SIGINT, {0x8067350, [INT], SA_RESTART}, {0x80d27f0, [INT], 
SA_RESTART}, 8) = 0
  select(6, [5], NULL, [5], {1, 0})       = 1 (in [5], left {0, 976000})
  recv(5, "$T4d0b:3ccd8dbe;0d:20cb8dbe;0f:d4"..., 8192, 0) = 54
  send(5, "+"..., 1, 0)                   = 1
  rt_sigaction(SIGINT, {0x80d27f0, [INT], SA_RESTART}, {0x8067350, [INT], 
SA_RESTART}, 8) = 0
  write(1, "\n"..., 1)                    = 1
  write(1, "Program received signal SIG32, Re"..., 51) = 51
  -----------------------------------------------------------------------


Here's the corrsponding section of the gdbserver strace

  ------------------------------gdbserver strace--------------------------
  read(4, "$vCont;c#a8"..., 4096)         = 11
  write(4, "+"..., 1)                     = 1
  rt_sigaction(SIGIO, {0xc608, [IO], SA_RESTART|0x4000000}, {0x1, [IO], 
SA_RESTART|0x4000000}, 8) = 0
  ptrace(PTRACE_PEEKTEXT, 791, 0x8920, [0xe28fc600]) = 0
  ptrace(PTRACE_GETREGS, 791, 0, 0x23a08) = 0
  ptrace(PTRACE_SETREGS, 791, 0, 0x23a08) = 0
  ptrace(PTRACE_CONT, 791, 0, SIG_0)      = 0
  --- SIGCHLD (Child exited) @ 0 (0) ---
  wait4(-1, [{WIFSTOPPED(s) && WSTOPSIG(s) == SIGTRAP} | 0x30000], WNOHANG, 
NULL) = 791
  ptrace(0x4201 /* PTRACE_??? */, 791, 0, 0xbefdcc10) = 0
  wait4(792, [{WIFSTOPPED(s) && WSTOPSIG(s) == SIGSTOP}], __WALL, NULL) = 792
  ptrace(0x4200 /* PTRACE_??? */, 792, 0, 0x8) = 0
  ptrace(PTRACE_CONT, 792, 0, SIG_0)      = 0
  ptrace(PTRACE_CONT, 791, 0, SIG_0)      = 0
  wait4(-1, 0xbefdcc14, WNOHANG, NULL)    = 0
  wait4(-1, 0xbefdcc14, WNOHANG|__WCLONE, NULL) = 0
  nanosleep({0, 1000000}, 0)              = ? ERESTART_RESTARTBLOCK (To be 
restarted)
  --- SIGCHLD (Child exited) @ 0 (0) ---
  restart_syscall(<... resuming interrupted call ...>) = 0
  wait4(-1, 0xbefdcc14, WNOHANG, NULL)    = 0
  wait4(-1, [{WIFSTOPPED(s) && WSTOPSIG(s) == SIGSTOP}], WNOHANG|__WCLONE, 
NULL) = 793
  wait4(-1, 0xbefdcc14, WNOHANG, NULL)    = 0
  wait4(-1, [{WIFSTOPPED(s) && WSTOPSIG(s) == SIGTRAP} | 0x30000], 
WNOHANG|__WCLONE, NULL) = 792
  ptrace(0x4201 /* PTRACE_??? */, 792, 0, 0xbefdcc10) = 0
  ptrace(0x4200 /* PTRACE_??? */, 793, 0, 0x8) = 0
  ptrace(PTRACE_CONT, 793, 0, SIG_0)      = 0
  ptrace(PTRACE_CONT, 792, 0, SIG_0)      = 0
  --- SIGCHLD (Child exited) @ 0 (0) ---
  wait4(-1, [{WIFSTOPPED(s) && WSTOPSIG(s) == SIGRTMIN}], WNOHANG, NULL) = 791
  syscall(0x9000ee, 0x318, 0x13, 0, 0x23b48) = 0
  --- SIGCHLD (Child exited) @ 0 (0) ---
  syscall(0x9000ee, 0x319, 0x13, 0, 0)    = 0
  --- SIGCHLD (Child exited) @ 0 (0) ---
  wait4(792, 0xbefdcbf4, WNOHANG, NULL)   = -1 ECHILD (No child processes)
  wait4(792, [{WIFSTOPPED(s) && WSTOPSIG(s) == SIGSTOP}], WNOHANG|__WCLONE, 
NULL) = 792
  wait4(793, 0xbefdcbf4, WNOHANG, NULL)   = -1 ECHILD (No child processes)
  wait4(793, [{WIFSTOPPED(s) && WSTOPSIG(s) == SIGSTOP}], WNOHANG|__WCLONE, 
NULL) = 793
  ptrace(PTRACE_GETREGS, 791, 0, 0x23c68) = 0
  rt_sigaction(SIGIO, {0x1, [IO], SA_RESTART|0x4000000}, {0xc608, [IO], 
SA_RESTART|0x4000000}, 8) = 0
  write(4, "$T4d0b:4cfdc6be;0d:30fbc6be;0f:d4"..., 54) = 54
  --- SIGIO (I/O possible) @ 0 (0) ---
  ------------------------------------------------------------------------

I'm trying to figure out which end is broken (gdb or
gdbserver), but I can't find any documetnation on what is
supposed to happen when a new thread is created.  My guess is
that gdbserver isn't working right, but that's just a hunch.

-- 
Grant Edwards                   grante             Yow! Hold the MAYO & pass
                                  at               the COSMIC AWARENESS ...
                               visi.com