From c4fd80fcd0c2cca60ae894ff38ec6460a9288231 Mon Sep 17 00:00:00 2001 From: Derek Parker Date: Mon, 23 Mar 2020 10:57:01 -0700 Subject: [PATCH] pkg/proc: Clean up proc.go This patch moves out unrelated types, variables and functions from proc.go into a place where they make more sense. --- go.sum | 1 + pkg/proc/bininfo.go | 156 ++++-- pkg/proc/breakpoints.go | 12 + pkg/proc/eval.go | 106 +++++ pkg/proc/gdbserial/gdbserver.go | 2 +- pkg/proc/interface.go | 2 +- pkg/proc/native/proc.go | 2 +- pkg/proc/proc.go | 812 ++++++++++++++++---------------- pkg/proc/target.go | 105 +++++ pkg/proc/threads.go | 551 ---------------------- pkg/proc/variables.go | 249 +++++++++- pkg/terminal/command_test.go | 2 +- 12 files changed, 1002 insertions(+), 998 deletions(-) diff --git a/go.sum b/go.sum index 7bcbbe16..fde29856 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ github.com/cosiner/argv v0.0.0-20170225145430-13bacc38a0a5 h1:rIXlvz2IWiupMFlC45cZCXZFvKX/ExBcSLrDy2G0Lp8= github.com/cosiner/argv v0.0.0-20170225145430-13bacc38a0a5/go.mod h1:p/NrK5tF6ICIly4qwEDsf6VDirFiWWz0FenfYBwJaKQ= +github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= diff --git a/pkg/proc/bininfo.go b/pkg/proc/bininfo.go index 244820b0..14a82004 100644 --- a/pkg/proc/bininfo.go +++ b/pkg/proc/bininfo.go @@ -34,6 +34,11 @@ import ( "github.com/sirupsen/logrus" ) +const ( + dwarfGoLanguage = 22 // DW_LANG_Go (from DWARF v5, section 7.12, page 231) + dwarfTreeCacheSize = 512 // size of the dwarfTree cache of each image +) + // BinaryInfo holds information on the binaries being executed (this // includes both the executable and also any loaded libraries). type BinaryInfo struct { @@ -102,13 +107,130 @@ type BinaryInfo struct { logger *logrus.Entry } -// ErrCouldNotDetermineRelocation is an error returned when Delve could not determine the base address of a -// position independant executable. -var ErrCouldNotDetermineRelocation = errors.New("could not determine the base address of a PIE") +var ( + // ErrCouldNotDetermineRelocation is an error returned when Delve could not determine the base address of a + // position independant executable. + ErrCouldNotDetermineRelocation = errors.New("could not determine the base address of a PIE") -// ErrNoDebugInfoFound is returned when Delve cannot open the debug_info -// section or find an external debug info file. -var ErrNoDebugInfoFound = errors.New("could not open debug info") + // ErrNoDebugInfoFound is returned when Delve cannot open the debug_info + // section or find an external debug info file. + ErrNoDebugInfoFound = errors.New("could not open debug info") +) + +var ( + supportedLinuxArch = map[elf.Machine]bool{ + elf.EM_X86_64: true, + elf.EM_AARCH64: true, + elf.EM_386: true, + } + + supportedWindowsArch = map[PEMachine]bool{ + IMAGE_FILE_MACHINE_AMD64: true, + } + + supportedDarwinArch = map[macho.Cpu]bool{ + macho.CpuAmd64: true, + } +) + +// ErrFunctionNotFound is returned when failing to find the +// function named 'FuncName' within the binary. +type ErrFunctionNotFound struct { + FuncName string +} + +func (err *ErrFunctionNotFound) Error() string { + return fmt.Sprintf("could not find function %s\n", err.FuncName) +} + +// FindFileLocation returns the PC for a given file:line. +// Assumes that `file` is normalized to lower case and '/' on Windows. +func FindFileLocation(p Process, fileName string, lineno int) ([]uint64, error) { + pcs, err := p.BinInfo().LineToPC(fileName, lineno) + if err != nil { + return nil, err + } + var fn *Function + for i := range pcs { + if fn == nil || pcs[i] < fn.Entry || pcs[i] >= fn.End { + fn = p.BinInfo().PCToFunc(pcs[i]) + } + if fn != nil && fn.Entry == pcs[i] { + pcs[i], _ = FirstPCAfterPrologue(p, fn, true) + } + } + return pcs, nil +} + +// FindFunctionLocation finds address of a function's line +// If lineOffset is passed FindFunctionLocation will return the address of that line +func FindFunctionLocation(p Process, funcName string, lineOffset int) ([]uint64, error) { + bi := p.BinInfo() + origfn := bi.LookupFunc[funcName] + if origfn == nil { + return nil, &ErrFunctionNotFound{funcName} + } + + if lineOffset <= 0 { + r := make([]uint64, 0, len(origfn.InlinedCalls)+1) + if origfn.Entry > 0 { + // add concrete implementation of the function + pc, err := FirstPCAfterPrologue(p, origfn, false) + if err != nil { + return nil, err + } + r = append(r, pc) + } + // add inlined calls to the function + for _, call := range origfn.InlinedCalls { + r = append(r, call.LowPC) + } + if len(r) == 0 { + return nil, &ErrFunctionNotFound{funcName} + } + return r, nil + } + filename, lineno := origfn.cu.lineInfo.PCToLine(origfn.Entry, origfn.Entry) + return bi.LineToPC(filename, lineno+lineOffset) +} + +// FirstPCAfterPrologue returns the address of the first +// instruction after the prologue for function fn. +// If sameline is set FirstPCAfterPrologue will always return an +// address associated with the same line as fn.Entry. +func FirstPCAfterPrologue(p Process, fn *Function, sameline bool) (uint64, error) { + pc, _, line, ok := fn.cu.lineInfo.PrologueEndPC(fn.Entry, fn.End) + if ok { + if !sameline { + return pc, nil + } + _, entryLine := fn.cu.lineInfo.PCToLine(fn.Entry, fn.Entry) + if entryLine == line { + return pc, nil + } + } + + pc, err := firstPCAfterPrologueDisassembly(p, fn, sameline) + if err != nil { + return fn.Entry, err + } + + if pc == fn.Entry { + // Look for the first instruction with the stmt flag set, so that setting a + // breakpoint with file:line and with the function name always result on + // the same instruction being selected. + if pc2, _, _, ok := fn.cu.lineInfo.FirstStmtForLine(fn.Entry, fn.End); ok { + return pc2, nil + } + } + + return pc, nil +} + +// CpuArch is a stringer interface representing CPU architectures. +type CpuArch interface { + String() string +} // ErrUnsupportedArch is returned when attempting to debug a binary compiled for an unsupported architecture. type ErrUnsupportedArch struct { @@ -116,10 +238,6 @@ type ErrUnsupportedArch struct { cpuArch CpuArch } -type CpuArch interface { - String() string -} - func (e *ErrUnsupportedArch) Error() string { var supportArchs []CpuArch switch e.os { @@ -151,24 +269,6 @@ func (e *ErrUnsupportedArch) Error() string { return errStr } -var supportedLinuxArch = map[elf.Machine]bool{ - elf.EM_X86_64: true, - elf.EM_AARCH64: true, - elf.EM_386: true, -} - -var supportedWindowsArch = map[PEMachine]bool{ - IMAGE_FILE_MACHINE_AMD64: true, -} - -var supportedDarwinArch = map[macho.Cpu]bool{ - macho.CpuAmd64: true, -} - -const dwarfGoLanguage = 22 // DW_LANG_Go (from DWARF v5, section 7.12, page 231) - -const dwarfTreeCacheSize = 512 // size of the dwarfTree cache of each image - type compileUnit struct { name string // univocal name for non-go compile units lowPC uint64 diff --git a/pkg/proc/breakpoints.go b/pkg/proc/breakpoints.go index 0be5b1aa..f3fcfb50 100644 --- a/pkg/proc/breakpoints.go +++ b/pkg/proc/breakpoints.go @@ -8,6 +8,18 @@ import ( "reflect" ) +const ( + // UnrecoveredPanic is the name given to the unrecovered panic breakpoint. + UnrecoveredPanic = "unrecovered-panic" + + // FatalThrow is the name given to the breakpoint triggered when the target + // process dies because of a fatal runtime error. + FatalThrow = "runtime-fatal-throw" + + unrecoveredPanicID = -1 + fatalThrowID = -2 +) + // Breakpoint represents a physical breakpoint. Stores information on the break // point including the byte of data that originally was stored at that // address. diff --git a/pkg/proc/eval.go b/pkg/proc/eval.go index cc23524d..175d4558 100644 --- a/pkg/proc/eval.go +++ b/pkg/proc/eval.go @@ -53,6 +53,112 @@ type EvalScope struct { callCtx *callContext } +// ConvertEvalScope returns a new EvalScope in the context of the +// specified goroutine ID and stack frame. +// If deferCall is > 0 the eval scope will be relative to the specified deferred call. +func ConvertEvalScope(dbp *Target, gid, frame, deferCall int) (*EvalScope, error) { + if _, err := dbp.Valid(); err != nil { + return nil, err + } + ct := dbp.CurrentThread() + g, err := FindGoroutine(dbp, gid) + if err != nil { + return nil, err + } + if g == nil { + return ThreadScope(ct) + } + + var thread MemoryReadWriter + if g.Thread == nil { + thread = ct + } else { + thread = g.Thread + } + + var opts StacktraceOptions + if deferCall > 0 { + opts = StacktraceReadDefers + } + + locs, err := g.Stacktrace(frame+1, opts) + if err != nil { + return nil, err + } + + if frame >= len(locs) { + return nil, fmt.Errorf("Frame %d does not exist in goroutine %d", frame, gid) + } + + if deferCall > 0 { + if deferCall-1 >= len(locs[frame].Defers) { + return nil, fmt.Errorf("Frame %d only has %d deferred calls", frame, len(locs[frame].Defers)) + } + + d := locs[frame].Defers[deferCall-1] + if d.Unreadable != nil { + return nil, d.Unreadable + } + + return d.EvalScope(ct) + } + + return FrameToScope(dbp.BinInfo(), thread, g, locs[frame:]...), nil +} + +// FrameToScope returns a new EvalScope for frames[0]. +// If frames has at least two elements all memory between +// frames[0].Regs.SP() and frames[1].Regs.CFA will be cached. +// Otherwise all memory between frames[0].Regs.SP() and frames[0].Regs.CFA +// will be cached. +func FrameToScope(bi *BinaryInfo, thread MemoryReadWriter, g *G, frames ...Stackframe) *EvalScope { + // Creates a cacheMem that will preload the entire stack frame the first + // time any local variable is read. + // Remember that the stack grows downward in memory. + minaddr := frames[0].Regs.SP() + var maxaddr uint64 + if len(frames) > 1 && frames[0].SystemStack == frames[1].SystemStack { + maxaddr = uint64(frames[1].Regs.CFA) + } else { + maxaddr = uint64(frames[0].Regs.CFA) + } + if maxaddr > minaddr && maxaddr-minaddr < maxFramePrefetchSize { + thread = cacheMemory(thread, uintptr(minaddr), int(maxaddr-minaddr)) + } + + s := &EvalScope{Location: frames[0].Call, Regs: frames[0].Regs, Mem: thread, g: g, BinInfo: bi, frameOffset: frames[0].FrameOffset()} + s.PC = frames[0].lastpc + return s +} + +// ThreadScope returns an EvalScope for the given thread. +func ThreadScope(thread Thread) (*EvalScope, error) { + locations, err := ThreadStacktrace(thread, 1) + if err != nil { + return nil, err + } + if len(locations) < 1 { + return nil, errors.New("could not decode first frame") + } + return FrameToScope(thread.BinInfo(), thread, nil, locations...), nil +} + +// GoroutineScope returns an EvalScope for the goroutine running on the given thread. +func GoroutineScope(thread Thread) (*EvalScope, error) { + locations, err := ThreadStacktrace(thread, 1) + if err != nil { + return nil, err + } + if len(locations) < 1 { + return nil, errors.New("could not decode first frame") + } + g, err := GetG(thread) + if err != nil { + return nil, err + } + return FrameToScope(thread.BinInfo(), thread, g, locations...), nil +} + // EvalExpression returns the value of the given expression. func (scope *EvalScope) EvalExpression(expr string, cfg LoadConfig) (*Variable, error) { if scope.callCtx != nil { diff --git a/pkg/proc/gdbserial/gdbserver.go b/pkg/proc/gdbserial/gdbserver.go index 19a6ccbf..85b651a6 100644 --- a/pkg/proc/gdbserial/gdbserver.go +++ b/pkg/proc/gdbserial/gdbserver.go @@ -574,7 +574,7 @@ func (p *Process) Pid() int { // and the process has not exited. func (p *Process) Valid() (bool, error) { if p.detached { - return false, &proc.ProcessDetachedError{} + return false, proc.ErrProcessDetached } if p.exited { return false, &proc.ErrProcessExited{Pid: p.Pid()} diff --git a/pkg/proc/interface.go b/pkg/proc/interface.go index 203a1253..d5f7a729 100644 --- a/pkg/proc/interface.go +++ b/pkg/proc/interface.go @@ -78,7 +78,7 @@ type Info interface { ResumeNotify(chan<- struct{}) // Valid returns true if this Process can be used. When it returns false it // also returns an error describing why the Process is invalid (either - // ErrProcessExited or ProcessDetachedError). + // ErrProcessExited or ErrProcessDetached). Valid() (bool, error) BinInfo() *BinaryInfo EntryPoint() (uint64, error) diff --git a/pkg/proc/native/proc.go b/pkg/proc/native/proc.go index 9299843d..22823a90 100644 --- a/pkg/proc/native/proc.go +++ b/pkg/proc/native/proc.go @@ -137,7 +137,7 @@ func (dbp *Process) Detach(kill bool) (err error) { // has not exited. func (dbp *Process) Valid() (bool, error) { if dbp.detached { - return false, &proc.ProcessDetachedError{} + return false, proc.ErrProcessDetached } if dbp.exited { return false, &proc.ErrProcessExited{Pid: dbp.Pid()} diff --git a/pkg/proc/proc.go b/pkg/proc/proc.go index 6a3607b8..5b831c17 100644 --- a/pkg/proc/proc.go +++ b/pkg/proc/proc.go @@ -5,114 +5,22 @@ import ( "errors" "fmt" "go/ast" - "go/constant" "go/token" - "os" + "path/filepath" "strconv" "strings" - "github.com/go-delve/delve/pkg/goversion" + "github.com/go-delve/delve/pkg/dwarf/reader" ) -// ErrNotExecutable is returned after attempting to execute a non-executable file -// to begin a debug session. -var ErrNotExecutable = errors.New("not an executable file") - -// ErrNotRecorded is returned when an action is requested that is -// only possible on recorded (traced) programs. -var ErrNotRecorded = errors.New("not a recording") - -var ErrNoRuntimeAllG = errors.New("could not find goroutine array") - -const ( - // UnrecoveredPanic is the name given to the unrecovered panic breakpoint. - UnrecoveredPanic = "unrecovered-panic" - - // FatalThrow is the name given to the breakpoint triggered when the target process dies because of a fatal runtime error - FatalThrow = "runtime-fatal-throw" - - unrecoveredPanicID = -1 - fatalThrowID = -2 -) - -// ErrProcessExited indicates that the process has exited and contains both -// process id and exit status. -type ErrProcessExited struct { - Pid int - Status int +// ErrNoSourceForPC is returned when the given address +// does not correspond with a source file location. +type ErrNoSourceForPC struct { + pc uint64 } -func (pe ErrProcessExited) Error() string { - return fmt.Sprintf("Process %d has exited with status %d", pe.Pid, pe.Status) -} - -// ProcessDetachedError indicates that we detached from the target process. -type ProcessDetachedError struct { -} - -func (pe ProcessDetachedError) Error() string { - return "detached from the process" -} - -// FindFileLocation returns the PC for a given file:line. -// Assumes that `file` is normalized to lower case and '/' on Windows. -func FindFileLocation(p Process, fileName string, lineno int) ([]uint64, error) { - pcs, err := p.BinInfo().LineToPC(fileName, lineno) - if err != nil { - return nil, err - } - var fn *Function - for i := range pcs { - if fn == nil || pcs[i] < fn.Entry || pcs[i] >= fn.End { - fn = p.BinInfo().PCToFunc(pcs[i]) - } - if fn != nil && fn.Entry == pcs[i] { - pcs[i], _ = FirstPCAfterPrologue(p, fn, true) - } - } - return pcs, nil -} - -// ErrFunctionNotFound is returned when failing to find the -// function named 'FuncName' within the binary. -type ErrFunctionNotFound struct { - FuncName string -} - -func (err *ErrFunctionNotFound) Error() string { - return fmt.Sprintf("Could not find function %s\n", err.FuncName) -} - -// FindFunctionLocation finds address of a function's line -// If lineOffset is passed FindFunctionLocation will return the address of that line -func FindFunctionLocation(p Process, funcName string, lineOffset int) ([]uint64, error) { - bi := p.BinInfo() - origfn := bi.LookupFunc[funcName] - if origfn == nil { - return nil, &ErrFunctionNotFound{funcName} - } - - if lineOffset <= 0 { - r := make([]uint64, 0, len(origfn.InlinedCalls)+1) - if origfn.Entry > 0 { - // add concrete implementation of the function - pc, err := FirstPCAfterPrologue(p, origfn, false) - if err != nil { - return nil, err - } - r = append(r, pc) - } - // add inlined calls to the function - for _, call := range origfn.InlinedCalls { - r = append(r, call.LowPC) - } - if len(r) == 0 { - return nil, &ErrFunctionNotFound{funcName} - } - return r, nil - } - filename, lineno := origfn.cu.lineInfo.PCToLine(origfn.Entry, origfn.Entry) - return bi.LineToPC(filename, lineno+lineOffset) +func (err *ErrNoSourceForPC) Error() string { + return fmt.Sprintf("no source for PC %#x", err.pc) } // Next continues execution until the next source line. @@ -236,7 +144,7 @@ func Continue(dbp *Target) error { // here we either set a breakpoint into the destination of the CALL // instruction or we determined that the called function is hidden, // either way we need to resume execution - if err = setStepIntoBreakpoint(dbp, text, SameGoroutineCondition(dbp.SelectedGoroutine())); err != nil { + if err = setStepIntoBreakpoint(dbp, text, sameGoroutineCondition(dbp.SelectedGoroutine())); err != nil { return err } } else { @@ -374,9 +282,9 @@ func Step(dbp *Target) (err error) { return Continue(dbp) } -// SameGoroutineCondition returns an expression that evaluates to true when +// sameGoroutineCondition returns an expression that evaluates to true when // the current goroutine is g. -func SameGoroutineCondition(g *G) ast.Expr { +func sameGoroutineCondition(g *G) ast.Expr { if g == nil { return nil } @@ -450,7 +358,7 @@ func StepOut(dbp *Target) error { return Continue(dbp) } - sameGCond := SameGoroutineCondition(selg) + sameGCond := sameGoroutineCondition(selg) retFrameCond := andFrameoffCondition(sameGCond, retframe.FrameOffset()) if backward { @@ -503,7 +411,7 @@ func StepInstruction(dbp *Target) (err error) { if g.Thread == nil { // Step called on parked goroutine if _, err := dbp.SetBreakpoint(g.PC, NextBreakpoint, - SameGoroutineCondition(dbp.SelectedGoroutine())); err != nil { + sameGoroutineCondition(dbp.SelectedGoroutine())); err != nil { return err } return Continue(dbp) @@ -529,6 +437,361 @@ func StepInstruction(dbp *Target) (err error) { return nil } +// Set breakpoints at every line, and the return address. Also look for +// a deferred function and set a breakpoint there too. +// If stepInto is true it will also set breakpoints inside all +// functions called on the current source line, for non-absolute CALLs +// a breakpoint of kind StepBreakpoint is set on the CALL instruction, +// Continue will take care of setting a breakpoint to the destination +// once the CALL is reached. +// +// Regardless of stepInto the following breakpoints will be set: +// - a breakpoint on the first deferred function with NextDeferBreakpoint +// kind, the list of all the addresses to deferreturn calls in this function +// and condition checking that we remain on the same goroutine +// - a breakpoint on each line of the function, with a condition checking +// that we stay on the same stack frame and goroutine. +// - a breakpoint on the return address of the function, with a condition +// checking that we move to the previous stack frame and stay on the same +// goroutine. +// +// The breakpoint on the return address is *not* set if the current frame is +// an inlined call. For inlined calls topframe.Current.Fn is the function +// where the inlining happened and the second set of breakpoints will also +// cover the "return address". +// +// If inlinedStepOut is true this function implements the StepOut operation +// for an inlined function call. Everything works the same as normal except +// when removing instructions belonging to inlined calls we also remove all +// instructions belonging to the current inlined call. +func next(dbp *Target, stepInto, inlinedStepOut bool) error { + backward := dbp.GetDirection() == Backward + selg := dbp.SelectedGoroutine() + curthread := dbp.CurrentThread() + topframe, retframe, err := topframe(selg, curthread) + if err != nil { + return err + } + + if topframe.Current.Fn == nil { + return &ErrNoSourceForPC{topframe.Current.PC} + } + + if backward && retframe.Current.Fn == nil { + return &ErrNoSourceForPC{retframe.Current.PC} + } + + // sanity check + if inlinedStepOut && !topframe.Inlined { + panic("next called with inlinedStepOut but topframe was not inlined") + } + + success := false + defer func() { + if !success { + dbp.ClearInternalBreakpoints() + } + }() + + ext := filepath.Ext(topframe.Current.File) + csource := ext != ".go" && ext != ".s" + var thread MemoryReadWriter = curthread + var regs Registers + if selg != nil && selg.Thread != nil { + thread = selg.Thread + regs, err = selg.Thread.Registers(false) + if err != nil { + return err + } + } + + sameGCond := sameGoroutineCondition(selg) + + var firstPCAfterPrologue uint64 + + if backward { + firstPCAfterPrologue, err = FirstPCAfterPrologue(dbp, topframe.Current.Fn, false) + if err != nil { + return err + } + if firstPCAfterPrologue == topframe.Current.PC { + // We don't want to step into the prologue so we just execute a reverse step out instead + if err := stepOutReverse(dbp, topframe, retframe, sameGCond); err != nil { + return err + } + + success = true + return nil + } + + topframe.Ret, err = findCallInstrForRet(dbp, thread, topframe.Ret, retframe.Current.Fn) + if err != nil { + return err + } + } + + text, err := disassemble(thread, regs, dbp.Breakpoints(), dbp.BinInfo(), topframe.Current.Fn.Entry, topframe.Current.Fn.End, false) + if err != nil && stepInto { + return err + } + + retFrameCond := andFrameoffCondition(sameGCond, retframe.FrameOffset()) + sameFrameCond := andFrameoffCondition(sameGCond, topframe.FrameOffset()) + var sameOrRetFrameCond ast.Expr + if sameGCond != nil { + if topframe.Inlined { + sameOrRetFrameCond = sameFrameCond + } else { + sameOrRetFrameCond = &ast.BinaryExpr{ + Op: token.LAND, + X: sameGCond, + Y: &ast.BinaryExpr{ + Op: token.LOR, + X: frameoffCondition(topframe.FrameOffset()), + Y: frameoffCondition(retframe.FrameOffset()), + }, + } + } + } + + if stepInto && !backward { + err := setStepIntoBreakpoints(dbp, text, topframe, sameGCond) + if err != nil { + return err + } + } + + if !backward { + _, err = setDeferBreakpoint(dbp, text, topframe, sameGCond, stepInto) + if err != nil { + return err + } + } + + // Add breakpoints on all the lines in the current function + pcs, err := topframe.Current.Fn.cu.lineInfo.AllPCsBetween(topframe.Current.Fn.Entry, topframe.Current.Fn.End-1, topframe.Current.File, topframe.Current.Line) + if err != nil { + return err + } + + if backward { + // Ensure that pcs contains firstPCAfterPrologue when reverse stepping. + found := false + for _, pc := range pcs { + if pc == firstPCAfterPrologue { + found = true + break + } + } + if !found { + pcs = append(pcs, firstPCAfterPrologue) + } + } + + if !stepInto { + // Removing any PC range belonging to an inlined call + frame := topframe + if inlinedStepOut { + frame = retframe + } + pcs, err = removeInlinedCalls(dbp, pcs, frame) + if err != nil { + return err + } + } + + if !csource { + var covered bool + for i := range pcs { + if topframe.Current.Fn.Entry <= pcs[i] && pcs[i] < topframe.Current.Fn.End { + covered = true + break + } + } + + if !covered { + fn := dbp.BinInfo().PCToFunc(topframe.Ret) + if selg != nil && fn != nil && fn.Name == "runtime.goexit" { + return nil + } + } + } + + for _, pc := range pcs { + if _, err := allowDuplicateBreakpoint(dbp.SetBreakpoint(pc, NextBreakpoint, sameFrameCond)); err != nil { + dbp.ClearInternalBreakpoints() + return err + } + } + + if stepInto && backward { + err := setStepIntoBreakpointsReverse(dbp, text, topframe, sameGCond) + if err != nil { + return err + } + } + + if !topframe.Inlined { + // Add a breakpoint on the return address for the current frame. + // For inlined functions there is no need to do this, the set of PCs + // returned by the AllPCsBetween call above already cover all instructions + // of the containing function. + bp, err := dbp.SetBreakpoint(topframe.Ret, NextBreakpoint, retFrameCond) + if _, isexists := err.(BreakpointExistsError); isexists { + if bp.Kind == NextBreakpoint { + // If the return address shares the same address with one of the lines + // of the function (because we are stepping through a recursive + // function) then the corresponding breakpoint should be active both on + // this frame and on the return frame. + bp.Cond = sameOrRetFrameCond + } + } + // Return address could be wrong, if we are unable to set a breakpoint + // there it's ok. + if bp != nil { + configureReturnBreakpoint(dbp.BinInfo(), bp, &topframe, retFrameCond) + } + } + + if bp := curthread.Breakpoint(); bp.Breakpoint == nil { + curthread.SetCurrentBreakpoint(false) + } + success = true + return nil +} + +func setStepIntoBreakpoints(dbp Process, text []AsmInstruction, topframe Stackframe, sameGCond ast.Expr) error { + for _, instr := range text { + if instr.Loc.File != topframe.Current.File || instr.Loc.Line != topframe.Current.Line || !instr.IsCall() { + continue + } + + if instr.DestLoc != nil { + if err := setStepIntoBreakpoint(dbp, []AsmInstruction{instr}, sameGCond); err != nil { + return err + } + } else { + // Non-absolute call instruction, set a StepBreakpoint here + if _, err := allowDuplicateBreakpoint(dbp.SetBreakpoint(instr.Loc.PC, StepBreakpoint, sameGCond)); err != nil { + return err + } + } + } + return nil +} + +func setStepIntoBreakpointsReverse(dbp Process, text []AsmInstruction, topframe Stackframe, sameGCond ast.Expr) error { + // Set a breakpoint after every CALL instruction + for i, instr := range text { + if instr.Loc.File != topframe.Current.File || !instr.IsCall() || instr.DestLoc == nil || instr.DestLoc.Fn == nil { + continue + } + + if fn := instr.DestLoc.Fn; strings.HasPrefix(fn.Name, "runtime.") && !isExportedRuntime(fn.Name) { + continue + } + + if nextIdx := i + 1; nextIdx < len(text) { + if _, err := allowDuplicateBreakpoint(dbp.SetBreakpoint(text[nextIdx].Loc.PC, StepBreakpoint, sameGCond)); err != nil { + return err + } + } + } + return nil +} + +func FindDeferReturnCalls(text []AsmInstruction) []uint64 { + const deferreturn = "runtime.deferreturn" + deferreturns := []uint64{} + + // Find all runtime.deferreturn locations in the function + // See documentation of Breakpoint.DeferCond for why this is necessary + for _, instr := range text { + if instr.IsCall() && instr.DestLoc != nil && instr.DestLoc.Fn != nil && instr.DestLoc.Fn.Name == deferreturn { + deferreturns = append(deferreturns, instr.Loc.PC) + } + } + return deferreturns +} + +// Removes instructions belonging to inlined calls of topframe from pcs. +// If includeCurrentFn is true it will also remove all instructions +// belonging to the current function. +func removeInlinedCalls(dbp Process, pcs []uint64, topframe Stackframe) ([]uint64, error) { + dwarfTree, err := topframe.Call.Fn.cu.image.getDwarfTree(topframe.Call.Fn.offset) + if err != nil { + return pcs, err + } + for _, e := range reader.InlineStack(dwarfTree, 0) { + if e.Offset == topframe.Call.Fn.offset { + continue + } + for _, rng := range e.Ranges { + pcs = removePCsBetween(pcs, rng[0], rng[1]) + } + } + return pcs, nil +} + +func removePCsBetween(pcs []uint64, start, end uint64) []uint64 { + out := pcs[:0] + for _, pc := range pcs { + if pc < start || pc >= end { + out = append(out, pc) + } + } + return out +} + +func setStepIntoBreakpoint(dbp Process, text []AsmInstruction, cond ast.Expr) error { + if len(text) <= 0 { + return nil + } + + instr := text[0] + + if instr.DestLoc == nil { + // Call destination couldn't be resolved because this was not the + // current instruction, therefore the step-into breakpoint can not be set. + return nil + } + + fn := instr.DestLoc.Fn + + // Skip unexported runtime functions + if fn != nil && strings.HasPrefix(fn.Name, "runtime.") && !isExportedRuntime(fn.Name) { + return nil + } + + //TODO(aarzilli): if we want to let users hide functions + // or entire packages from being stepped into with 'step' + // those extra checks should be done here. + + pc := instr.DestLoc.PC + + // Skip InhibitStepInto functions for different arch. + if dbp.BinInfo().Arch.InhibitStepInto(dbp.BinInfo(), pc) { + return nil + } + + // We want to skip the function prologue but we should only do it if the + // destination address of the CALL instruction is the entry point of the + // function. + // Calls to runtime.duffzero and duffcopy inserted by the compiler can + // sometimes point inside the body of those functions, well after the + // prologue. + if fn != nil && fn.Entry == instr.DestLoc.PC { + pc, _ = FirstPCAfterPrologue(dbp, fn, false) + } + + // Set a breakpoint after the function's prologue + if _, err := allowDuplicateBreakpoint(dbp.SetBreakpoint(pc, NextBreakpoint, cond)); err != nil { + return err + } + + return nil +} + func allowDuplicateBreakpoint(bp *Breakpoint, err error) (*Breakpoint, error) { if err != nil { if _, isexists := err.(BreakpointExistsError); isexists { @@ -642,321 +905,42 @@ func stepOutReverse(p *Target, topframe, retframe Stackframe, sameGCond ast.Expr return err } -// GoroutinesInfo searches for goroutines starting at index 'start', and -// returns an array of up to 'count' (or all found elements, if 'count' is 0) -// G structures representing the information Delve care about from the internal -// runtime G structure. -// GoroutinesInfo also returns the next index to be used as 'start' argument -// while scanning for all available goroutines, or -1 if there was an error -// or if the index already reached the last possible value. -func GoroutinesInfo(dbp *Target, start, count int) ([]*G, int, error) { - if _, err := dbp.Valid(); err != nil { - return nil, -1, err - } - if dbp.gcache.allGCache != nil { - // We can't use the cached array to fulfill a subrange request - if start == 0 && (count == 0 || count >= len(dbp.gcache.allGCache)) { - return dbp.gcache.allGCache, -1, nil +// onNextGoroutine returns true if this thread is on the goroutine requested by the current 'next' command +func onNextGoroutine(thread Thread, breakpoints *BreakpointMap) (bool, error) { + var bp *Breakpoint + for i := range breakpoints.M { + if breakpoints.M[i].Kind != UserBreakpoint && breakpoints.M[i].internalCond != nil { + bp = breakpoints.M[i] + break } } - - var ( - threadg = map[int]*G{} - allg []*G - ) - - threads := dbp.ThreadList() - for _, th := range threads { - if th.Blocked() { - continue - } - g, _ := GetG(th) - if g != nil { - threadg[g.ID] = g - } + if bp == nil { + return false, nil } - - allgptr, allglen, err := dbp.gcache.getRuntimeAllg(dbp.BinInfo(), dbp.CurrentThread()) - if err != nil { - return nil, -1, err - } - - for i := uint64(start); i < allglen; i++ { - if count != 0 && len(allg) >= count { - return allg, int(i), nil - } - gvar, err := newGVariable(dbp.CurrentThread(), uintptr(allgptr+(i*uint64(dbp.BinInfo().Arch.PtrSize()))), true) - if err != nil { - allg = append(allg, &G{Unreadable: err}) - continue - } - g, err := gvar.parseG() - if err != nil { - allg = append(allg, &G{Unreadable: err}) - continue - } - if thg, allocated := threadg[g.ID]; allocated { - loc, err := thg.Thread.Location() - if err != nil { - return nil, -1, err - } - g.Thread = thg.Thread - // Prefer actual thread location information. - g.CurrentLoc = *loc - g.SystemStack = thg.SystemStack - } - if g.Status != Gdead { - allg = append(allg, g) - } - dbp.gcache.addGoroutine(g) - } - if start == 0 { - dbp.gcache.allGCache = allg - } - - return allg, -1, nil + // Internal breakpoint conditions can take multiple different forms: + // Step into breakpoints: + // runtime.curg.goid == X + // Next or StepOut breakpoints: + // runtime.curg.goid == X && runtime.frameoff == Y + // Breakpoints that can be hit either by stepping on a line in the same + // function or by returning from the function: + // runtime.curg.goid == X && (runtime.frameoff == Y || runtime.frameoff == Z) + // Here we are only interested in testing the runtime.curg.goid clause. + w := onNextGoroutineWalker{thread: thread} + ast.Walk(&w, bp.internalCond) + return w.ret, w.err } -// FindGoroutine returns a G struct representing the goroutine -// specified by `gid`. -func FindGoroutine(dbp *Target, gid int) (*G, error) { - if selg := dbp.SelectedGoroutine(); (gid == -1) || (selg != nil && selg.ID == gid) || (selg == nil && gid == 0) { - // Return the currently selected goroutine in the following circumstances: - // - // 1. if the caller asks for gid == -1 (because that's what a goroutine ID of -1 means in our API). - // 2. if gid == selg.ID. - // this serves two purposes: (a) it's an optimizations that allows us - // to avoid reading any other goroutine and, more importantly, (b) we - // could be reading an incorrect value for the goroutine ID of a thread. - // This condition usually happens when a goroutine calls runtime.clone - // and for a short period of time two threads will appear to be running - // the same goroutine. - // 3. if the caller asks for gid == 0 and the selected goroutine is - // either 0 or nil. - // Goroutine 0 is special, it either means we have no current goroutine - // (for example, running C code), or that we are running on a speical - // stack (system stack, signal handling stack) and we didn't properly - // detect it. - // Since there could be multiple goroutines '0' running simultaneously - // if the user requests it return the one that's already selected or - // nil if there isn't a selected goroutine. - return selg, nil - } - - if gid == 0 { - return nil, fmt.Errorf("Unknown goroutine %d", gid) - } - - // Calling GoroutinesInfo could be slow if there are many goroutines - // running, check if a running goroutine has been requested first. - for _, thread := range dbp.ThreadList() { - g, _ := GetG(thread) - if g != nil && g.ID == gid { - return g, nil - } - } - - if g := dbp.gcache.partialGCache[gid]; g != nil { - return g, nil - } - - const goroutinesInfoLimit = 10 - nextg := 0 - for nextg >= 0 { - var gs []*G - var err error - gs, nextg, err = GoroutinesInfo(dbp, nextg, goroutinesInfoLimit) - if err != nil { - return nil, err - } - for i := range gs { - if gs[i].ID == gid { - if gs[i].Unreadable != nil { - return nil, gs[i].Unreadable - } - return gs[i], nil - } - } - } - - return nil, fmt.Errorf("Unknown goroutine %d", gid) +type onNextGoroutineWalker struct { + thread Thread + ret bool + err error } -// ConvertEvalScope returns a new EvalScope in the context of the -// specified goroutine ID and stack frame. -// If deferCall is > 0 the eval scope will be relative to the specified deferred call. -func ConvertEvalScope(dbp *Target, gid, frame, deferCall int) (*EvalScope, error) { - if _, err := dbp.Valid(); err != nil { - return nil, err +func (w *onNextGoroutineWalker) Visit(n ast.Node) ast.Visitor { + if binx, isbin := n.(*ast.BinaryExpr); isbin && binx.Op == token.EQL && exprToString(binx.X) == "runtime.curg.goid" { + w.ret, w.err = evalBreakpointCondition(w.thread, n.(ast.Expr)) + return nil } - ct := dbp.CurrentThread() - g, err := FindGoroutine(dbp, gid) - if err != nil { - return nil, err - } - if g == nil { - return ThreadScope(ct) - } - - var thread MemoryReadWriter - if g.Thread == nil { - thread = ct - } else { - thread = g.Thread - } - - var opts StacktraceOptions - if deferCall > 0 { - opts = StacktraceReadDefers - } - - locs, err := g.Stacktrace(frame+1, opts) - if err != nil { - return nil, err - } - - if frame >= len(locs) { - return nil, fmt.Errorf("Frame %d does not exist in goroutine %d", frame, gid) - } - - if deferCall > 0 { - if deferCall-1 >= len(locs[frame].Defers) { - return nil, fmt.Errorf("Frame %d only has %d deferred calls", frame, len(locs[frame].Defers)) - } - - d := locs[frame].Defers[deferCall-1] - if d.Unreadable != nil { - return nil, d.Unreadable - } - - return d.EvalScope(ct) - } - - return FrameToScope(dbp.BinInfo(), thread, g, locs[frame:]...), nil -} - -// FrameToScope returns a new EvalScope for frames[0]. -// If frames has at least two elements all memory between -// frames[0].Regs.SP() and frames[1].Regs.CFA will be cached. -// Otherwise all memory between frames[0].Regs.SP() and frames[0].Regs.CFA -// will be cached. -func FrameToScope(bi *BinaryInfo, thread MemoryReadWriter, g *G, frames ...Stackframe) *EvalScope { - // Creates a cacheMem that will preload the entire stack frame the first - // time any local variable is read. - // Remember that the stack grows downward in memory. - minaddr := frames[0].Regs.SP() - var maxaddr uint64 - if len(frames) > 1 && frames[0].SystemStack == frames[1].SystemStack { - maxaddr = uint64(frames[1].Regs.CFA) - } else { - maxaddr = uint64(frames[0].Regs.CFA) - } - if maxaddr > minaddr && maxaddr-minaddr < maxFramePrefetchSize { - thread = cacheMemory(thread, uintptr(minaddr), int(maxaddr-minaddr)) - } - - s := &EvalScope{Location: frames[0].Call, Regs: frames[0].Regs, Mem: thread, g: g, BinInfo: bi, frameOffset: frames[0].FrameOffset()} - s.PC = frames[0].lastpc - return s -} - -// createUnrecoveredPanicBreakpoint creates the unrecoverable-panic breakpoint. -// This function is meant to be called by implementations of the Process interface. -func createUnrecoveredPanicBreakpoint(p Process, writeBreakpoint WriteBreakpointFn) { - panicpcs, err := FindFunctionLocation(p, "runtime.startpanic", 0) - if _, isFnNotFound := err.(*ErrFunctionNotFound); isFnNotFound { - panicpcs, err = FindFunctionLocation(p, "runtime.fatalpanic", 0) - } - if err == nil { - bp, err := p.Breakpoints().SetWithID(unrecoveredPanicID, panicpcs[0], writeBreakpoint) - if err == nil { - bp.Name = UnrecoveredPanic - bp.Variables = []string{"runtime.curg._panic.arg"} - } - } -} - -func createFatalThrowBreakpoint(p Process, writeBreakpoint WriteBreakpointFn) { - fatalpcs, err := FindFunctionLocation(p, "runtime.fatalthrow", 0) - if err == nil { - bp, err := p.Breakpoints().SetWithID(fatalThrowID, fatalpcs[0], writeBreakpoint) - if err == nil { - bp.Name = FatalThrow - } - } -} - -// FirstPCAfterPrologue returns the address of the first -// instruction after the prologue for function fn. -// If sameline is set FirstPCAfterPrologue will always return an -// address associated with the same line as fn.Entry. -func FirstPCAfterPrologue(p Process, fn *Function, sameline bool) (uint64, error) { - pc, _, line, ok := fn.cu.lineInfo.PrologueEndPC(fn.Entry, fn.End) - if ok { - if !sameline { - return pc, nil - } - _, entryLine := fn.cu.lineInfo.PCToLine(fn.Entry, fn.Entry) - if entryLine == line { - return pc, nil - } - } - - pc, err := firstPCAfterPrologueDisassembly(p, fn, sameline) - if err != nil { - return fn.Entry, err - } - - if pc == fn.Entry { - // Look for the first instruction with the stmt flag set, so that setting a - // breakpoint with file:line and with the function name always result on - // the same instruction being selected. - if pc2, _, _, ok := fn.cu.lineInfo.FirstStmtForLine(fn.Entry, fn.End); ok { - return pc2, nil - } - } - - return pc, nil -} - -func setAsyncPreemptOff(p *Target, v int64) { - logger := p.BinInfo().logger - if producer := p.BinInfo().Producer(); producer == "" || !goversion.ProducerAfterOrEqual(producer, 1, 14) { - return - } - scope := globalScope(p.BinInfo(), p.BinInfo().Images[0], p.CurrentThread()) - debugv, err := scope.findGlobal("runtime", "debug") - if err != nil || debugv.Unreadable != nil { - logger.Warnf("could not find runtime/debug variable (or unreadable): %v %v", err, debugv.Unreadable) - return - } - asyncpreemptoffv, err := debugv.structMember("asyncpreemptoff") - if err != nil { - logger.Warnf("could not find asyncpreemptoff field: %v", err) - return - } - asyncpreemptoffv.loadValue(loadFullValue) - if asyncpreemptoffv.Unreadable != nil { - logger.Warnf("asyncpreemptoff field unreadable: %v", asyncpreemptoffv.Unreadable) - return - } - p.asyncPreemptChanged = true - p.asyncPreemptOff, _ = constant.Int64Val(asyncpreemptoffv.Value) - - err = scope.setValue(asyncpreemptoffv, newConstant(constant.MakeInt64(v), scope.Mem), "") - logger.Warnf("could not set asyncpreemptoff %v", err) -} - -// DisableAsyncPreemptEnv returns a process environment (like os.Environ) -// where asyncpreemptoff is set to 1. -func DisableAsyncPreemptEnv() []string { - env := os.Environ() - for i := range env { - if strings.HasPrefix(env[i], "GODEBUG=") { - // Go 1.14 asynchronous preemption mechanism is incompatible with - // debuggers, see: https://github.com/golang/go/issues/36494 - env[i] += ",asyncpreemptoff=1" - } - } - return env + return w } diff --git a/pkg/proc/target.go b/pkg/proc/target.go index f56ee242..3ba3de07 100644 --- a/pkg/proc/target.go +++ b/pkg/proc/target.go @@ -1,7 +1,30 @@ package proc import ( + "errors" "fmt" + "go/constant" + "os" + "strings" + + "github.com/go-delve/delve/pkg/goversion" +) + +var ( + // ErrNotExecutable is returned after attempting to execute a non-executable file + // to begin a debug session. + ErrNotExecutable = errors.New("not an executable file") + + // ErrNotRecorded is returned when an action is requested that is + // only possible on recorded (traced) programs. + ErrNotRecorded = errors.New("not a recording") + + // ErrNoRuntimeAllG is returned when the runtime.allg list could + // not be found. + ErrNoRuntimeAllG = errors.New("could not find goroutine array") + + // ErrProcessDetached indicates that we detached from the target process. + ErrProcessDetached = errors.New("detached from the process") ) // Target represents the process being debugged. @@ -31,6 +54,17 @@ type Target struct { gcache goroutineCache } +// ErrProcessExited indicates that the process has exited and contains both +// process id and exit status. +type ErrProcessExited struct { + Pid int + Status int +} + +func (pe ErrProcessExited) Error() string { + return fmt.Sprintf("Process %d has exited with status %d", pe.Pid, pe.Status) +} + // StopReason describes the reason why the target process is stopped. // A process could be stopped for multiple simultaneous reasons, in which // case only one will be reported. @@ -57,6 +91,20 @@ type NewTargetConfig struct { StopReason StopReason // Initial stop reason } +// DisableAsyncPreemptEnv returns a process environment (like os.Environ) +// where asyncpreemptoff is set to 1. +func DisableAsyncPreemptEnv() []string { + env := os.Environ() + for i := range env { + if strings.HasPrefix(env[i], "GODEBUG=") { + // Go 1.14 asynchronous preemption mechanism is incompatible with + // debuggers, see: https://github.com/golang/go/issues/36494 + env[i] += ",asyncpreemptoff=1" + } + } + return env +} + // NewTarget returns an initialized Target object. func NewTarget(p Process, cfg NewTargetConfig) (*Target, error) { entryPoint, err := p.EntryPoint() @@ -179,3 +227,60 @@ func (t *Target) Detach(kill bool) error { t.StopReason = StopUnknown return t.proc.Detach(kill) } + +// setAsyncPreemptOff enables or disables async goroutine preemption by +// writing the value 'v' to runtime.debug.asyncpreemptoff. +// A value of '1' means off, a value of '0' means on. +func setAsyncPreemptOff(p *Target, v int64) { + if producer := p.BinInfo().Producer(); producer == "" || !goversion.ProducerAfterOrEqual(producer, 1, 14) { + return + } + logger := p.BinInfo().logger + scope := globalScope(p.BinInfo(), p.BinInfo().Images[0], p.CurrentThread()) + debugv, err := scope.findGlobal("runtime", "debug") + if err != nil || debugv.Unreadable != nil { + logger.Warnf("could not find runtime/debug variable (or unreadable): %v %v", err, debugv.Unreadable) + return + } + asyncpreemptoffv, err := debugv.structMember("asyncpreemptoff") + if err != nil { + logger.Warnf("could not find asyncpreemptoff field: %v", err) + return + } + asyncpreemptoffv.loadValue(loadFullValue) + if asyncpreemptoffv.Unreadable != nil { + logger.Warnf("asyncpreemptoff field unreadable: %v", asyncpreemptoffv.Unreadable) + return + } + p.asyncPreemptChanged = true + p.asyncPreemptOff, _ = constant.Int64Val(asyncpreemptoffv.Value) + + err = scope.setValue(asyncpreemptoffv, newConstant(constant.MakeInt64(v), scope.Mem), "") + logger.Warnf("could not set asyncpreemptoff %v", err) +} + +// createUnrecoveredPanicBreakpoint creates the unrecoverable-panic breakpoint. +func createUnrecoveredPanicBreakpoint(p Process, writeBreakpoint WriteBreakpointFn) { + panicpcs, err := FindFunctionLocation(p, "runtime.startpanic", 0) + if _, isFnNotFound := err.(*ErrFunctionNotFound); isFnNotFound { + panicpcs, err = FindFunctionLocation(p, "runtime.fatalpanic", 0) + } + if err == nil { + bp, err := p.Breakpoints().SetWithID(unrecoveredPanicID, panicpcs[0], writeBreakpoint) + if err == nil { + bp.Name = UnrecoveredPanic + bp.Variables = []string{"runtime.curg._panic.arg"} + } + } +} + +// createFatalThrowBreakpoint creates the a breakpoint as runtime.fatalthrow. +func createFatalThrowBreakpoint(p Process, writeBreakpoint WriteBreakpointFn) { + fatalpcs, err := FindFunctionLocation(p, "runtime.fatalthrow", 0) + if err == nil { + bp, err := p.Breakpoints().SetWithID(fatalThrowID, fatalpcs[0], writeBreakpoint) + if err == nil { + bp.Name = FatalThrow + } + } +} diff --git a/pkg/proc/threads.go b/pkg/proc/threads.go index cfa9a157..4422835c 100644 --- a/pkg/proc/threads.go +++ b/pkg/proc/threads.go @@ -2,15 +2,6 @@ package proc import ( "errors" - "fmt" - "go/ast" - "go/token" - "path/filepath" - "reflect" - "strings" - - "github.com/go-delve/delve/pkg/dwarf/godwarf" - "github.com/go-delve/delve/pkg/dwarf/reader" ) // Thread represents a thread. @@ -104,545 +95,3 @@ func topframe(g *G, thread Thread) (Stackframe, Stackframe, error) { return frames[0], frames[1], nil } } - -// ErrNoSourceForPC is returned when the given address -// does not correspond with a source file location. -type ErrNoSourceForPC struct { - pc uint64 -} - -func (err *ErrNoSourceForPC) Error() string { - return fmt.Sprintf("no source for PC %#x", err.pc) -} - -// Set breakpoints at every line, and the return address. Also look for -// a deferred function and set a breakpoint there too. -// If stepInto is true it will also set breakpoints inside all -// functions called on the current source line, for non-absolute CALLs -// a breakpoint of kind StepBreakpoint is set on the CALL instruction, -// Continue will take care of setting a breakpoint to the destination -// once the CALL is reached. -// -// Regardless of stepInto the following breakpoints will be set: -// - a breakpoint on the first deferred function with NextDeferBreakpoint -// kind, the list of all the addresses to deferreturn calls in this function -// and condition checking that we remain on the same goroutine -// - a breakpoint on each line of the function, with a condition checking -// that we stay on the same stack frame and goroutine. -// - a breakpoint on the return address of the function, with a condition -// checking that we move to the previous stack frame and stay on the same -// goroutine. -// -// The breakpoint on the return address is *not* set if the current frame is -// an inlined call. For inlined calls topframe.Current.Fn is the function -// where the inlining happened and the second set of breakpoints will also -// cover the "return address". -// -// If inlinedStepOut is true this function implements the StepOut operation -// for an inlined function call. Everything works the same as normal except -// when removing instructions belonging to inlined calls we also remove all -// instructions belonging to the current inlined call. -func next(dbp *Target, stepInto, inlinedStepOut bool) error { - backward := dbp.GetDirection() == Backward - selg := dbp.SelectedGoroutine() - curthread := dbp.CurrentThread() - topframe, retframe, err := topframe(selg, curthread) - if err != nil { - return err - } - - if topframe.Current.Fn == nil { - return &ErrNoSourceForPC{topframe.Current.PC} - } - - if backward && retframe.Current.Fn == nil { - return &ErrNoSourceForPC{retframe.Current.PC} - } - - // sanity check - if inlinedStepOut && !topframe.Inlined { - panic("next called with inlinedStepOut but topframe was not inlined") - } - - success := false - defer func() { - if !success { - dbp.ClearInternalBreakpoints() - } - }() - - ext := filepath.Ext(topframe.Current.File) - csource := ext != ".go" && ext != ".s" - var thread MemoryReadWriter = curthread - var regs Registers - if selg != nil && selg.Thread != nil { - thread = selg.Thread - regs, err = selg.Thread.Registers(false) - if err != nil { - return err - } - } - - sameGCond := SameGoroutineCondition(selg) - - var firstPCAfterPrologue uint64 - - if backward { - firstPCAfterPrologue, err = FirstPCAfterPrologue(dbp, topframe.Current.Fn, false) - if err != nil { - return err - } - if firstPCAfterPrologue == topframe.Current.PC { - // We don't want to step into the prologue so we just execute a reverse step out instead - if err := stepOutReverse(dbp, topframe, retframe, sameGCond); err != nil { - return err - } - - success = true - return nil - } - - topframe.Ret, err = findCallInstrForRet(dbp, thread, topframe.Ret, retframe.Current.Fn) - if err != nil { - return err - } - } - - text, err := disassemble(thread, regs, dbp.Breakpoints(), dbp.BinInfo(), topframe.Current.Fn.Entry, topframe.Current.Fn.End, false) - if err != nil && stepInto { - return err - } - - retFrameCond := andFrameoffCondition(sameGCond, retframe.FrameOffset()) - sameFrameCond := andFrameoffCondition(sameGCond, topframe.FrameOffset()) - var sameOrRetFrameCond ast.Expr - if sameGCond != nil { - if topframe.Inlined { - sameOrRetFrameCond = sameFrameCond - } else { - sameOrRetFrameCond = &ast.BinaryExpr{ - Op: token.LAND, - X: sameGCond, - Y: &ast.BinaryExpr{ - Op: token.LOR, - X: frameoffCondition(topframe.FrameOffset()), - Y: frameoffCondition(retframe.FrameOffset()), - }, - } - } - } - - if stepInto && !backward { - err := setStepIntoBreakpoints(dbp, text, topframe, sameGCond) - if err != nil { - return err - } - } - - if !backward { - _, err = setDeferBreakpoint(dbp, text, topframe, sameGCond, stepInto) - if err != nil { - return err - } - } - - // Add breakpoints on all the lines in the current function - pcs, err := topframe.Current.Fn.cu.lineInfo.AllPCsBetween(topframe.Current.Fn.Entry, topframe.Current.Fn.End-1, topframe.Current.File, topframe.Current.Line) - if err != nil { - return err - } - - if backward { - // Ensure that pcs contains firstPCAfterPrologue when reverse stepping. - found := false - for _, pc := range pcs { - if pc == firstPCAfterPrologue { - found = true - break - } - } - if !found { - pcs = append(pcs, firstPCAfterPrologue) - } - } - - if !stepInto { - // Removing any PC range belonging to an inlined call - frame := topframe - if inlinedStepOut { - frame = retframe - } - pcs, err = removeInlinedCalls(dbp, pcs, frame) - if err != nil { - return err - } - } - - if !csource { - var covered bool - for i := range pcs { - if topframe.Current.Fn.Entry <= pcs[i] && pcs[i] < topframe.Current.Fn.End { - covered = true - break - } - } - - if !covered { - fn := dbp.BinInfo().PCToFunc(topframe.Ret) - if selg != nil && fn != nil && fn.Name == "runtime.goexit" { - return nil - } - } - } - - for _, pc := range pcs { - if _, err := allowDuplicateBreakpoint(dbp.SetBreakpoint(pc, NextBreakpoint, sameFrameCond)); err != nil { - dbp.ClearInternalBreakpoints() - return err - } - } - - if stepInto && backward { - err := setStepIntoBreakpointsReverse(dbp, text, topframe, sameGCond) - if err != nil { - return err - } - } - - if !topframe.Inlined { - // Add a breakpoint on the return address for the current frame. - // For inlined functions there is no need to do this, the set of PCs - // returned by the AllPCsBetween call above already cover all instructions - // of the containing function. - bp, err := dbp.SetBreakpoint(topframe.Ret, NextBreakpoint, retFrameCond) - if err != nil { - if _, isexists := err.(BreakpointExistsError); isexists { - if bp.Kind == NextBreakpoint { - // If the return address shares the same address with one of the lines - // of the function (because we are stepping through a recursive - // function) then the corresponding breakpoint should be active both on - // this frame and on the return frame. - bp.Cond = sameOrRetFrameCond - } - } - // Return address could be wrong, if we are unable to set a breakpoint - // there it's ok. - } - if bp != nil { - configureReturnBreakpoint(dbp.BinInfo(), bp, &topframe, retFrameCond) - } - } - - if bp := curthread.Breakpoint(); bp.Breakpoint == nil { - curthread.SetCurrentBreakpoint(false) - } - success = true - return nil -} - -func setStepIntoBreakpoints(dbp Process, text []AsmInstruction, topframe Stackframe, sameGCond ast.Expr) error { - for _, instr := range text { - if instr.Loc.File != topframe.Current.File || instr.Loc.Line != topframe.Current.Line || !instr.IsCall() { - continue - } - - if instr.DestLoc != nil { - if err := setStepIntoBreakpoint(dbp, []AsmInstruction{instr}, sameGCond); err != nil { - return err - } - } else { - // Non-absolute call instruction, set a StepBreakpoint here - if _, err := allowDuplicateBreakpoint(dbp.SetBreakpoint(instr.Loc.PC, StepBreakpoint, sameGCond)); err != nil { - return err - } - } - } - return nil -} - -func setStepIntoBreakpointsReverse(dbp Process, text []AsmInstruction, topframe Stackframe, sameGCond ast.Expr) error { - // Set a breakpoint after every CALL instruction - for i, instr := range text { - if instr.Loc.File != topframe.Current.File || !instr.IsCall() || instr.DestLoc == nil || instr.DestLoc.Fn == nil { - continue - } - - if fn := instr.DestLoc.Fn; strings.HasPrefix(fn.Name, "runtime.") && !isExportedRuntime(fn.Name) { - continue - } - - if nextIdx := i + 1; nextIdx < len(text) { - if _, err := allowDuplicateBreakpoint(dbp.SetBreakpoint(text[nextIdx].Loc.PC, StepBreakpoint, sameGCond)); err != nil { - return err - } - } - } - return nil -} - -func FindDeferReturnCalls(text []AsmInstruction) []uint64 { - const deferreturn = "runtime.deferreturn" - deferreturns := []uint64{} - - // Find all runtime.deferreturn locations in the function - // See documentation of Breakpoint.DeferCond for why this is necessary - for _, instr := range text { - if instr.IsCall() && instr.DestLoc != nil && instr.DestLoc.Fn != nil && instr.DestLoc.Fn.Name == deferreturn { - deferreturns = append(deferreturns, instr.Loc.PC) - } - } - return deferreturns -} - -// Removes instructions belonging to inlined calls of topframe from pcs. -// If includeCurrentFn is true it will also remove all instructions -// belonging to the current function. -func removeInlinedCalls(dbp Process, pcs []uint64, topframe Stackframe) ([]uint64, error) { - dwarfTree, err := topframe.Call.Fn.cu.image.getDwarfTree(topframe.Call.Fn.offset) - if err != nil { - return pcs, err - } - for _, e := range reader.InlineStack(dwarfTree, 0) { - if e.Offset == topframe.Call.Fn.offset { - continue - } - for _, rng := range e.Ranges { - pcs = removePCsBetween(pcs, rng[0], rng[1]) - } - } - return pcs, nil -} - -func removePCsBetween(pcs []uint64, start, end uint64) []uint64 { - out := pcs[:0] - for _, pc := range pcs { - if pc < start || pc >= end { - out = append(out, pc) - } - } - return out -} - -func setStepIntoBreakpoint(dbp Process, text []AsmInstruction, cond ast.Expr) error { - if len(text) <= 0 { - return nil - } - - instr := text[0] - - if instr.DestLoc == nil { - // Call destination couldn't be resolved because this was not the - // current instruction, therefore the step-into breakpoint can not be set. - return nil - } - - fn := instr.DestLoc.Fn - - // Skip unexported runtime functions - if fn != nil && strings.HasPrefix(fn.Name, "runtime.") && !isExportedRuntime(fn.Name) { - return nil - } - - //TODO(aarzilli): if we want to let users hide functions - // or entire packages from being stepped into with 'step' - // those extra checks should be done here. - - pc := instr.DestLoc.PC - - // Skip InhibitStepInto functions for different arch. - if dbp.BinInfo().Arch.InhibitStepInto(dbp.BinInfo(), pc) { - return nil - } - - // We want to skip the function prologue but we should only do it if the - // destination address of the CALL instruction is the entry point of the - // function. - // Calls to runtime.duffzero and duffcopy inserted by the compiler can - // sometimes point inside the body of those functions, well after the - // prologue. - if fn != nil && fn.Entry == instr.DestLoc.PC { - pc, _ = FirstPCAfterPrologue(dbp, fn, false) - } - - // Set a breakpoint after the function's prologue - if _, err := allowDuplicateBreakpoint(dbp.SetBreakpoint(pc, NextBreakpoint, cond)); err != nil { - return err - } - - return nil -} - -func getGVariable(thread Thread) (*Variable, error) { - regs, err := thread.Registers(false) - if err != nil { - return nil, err - } - - gaddr, hasgaddr := regs.GAddr() - if !hasgaddr { - var err error - gaddr, err = readUintRaw(thread, uintptr(regs.TLS()+thread.BinInfo().GStructOffset()), int64(thread.BinInfo().Arch.PtrSize())) - if err != nil { - return nil, err - } - } - - return newGVariable(thread, uintptr(gaddr), thread.Arch().DerefTLS()) -} - -func newGVariable(thread Thread, gaddr uintptr, deref bool) (*Variable, error) { - typ, err := thread.BinInfo().findType("runtime.g") - if err != nil { - return nil, err - } - - name := "" - - if deref { - typ = &godwarf.PtrType{ - CommonType: godwarf.CommonType{ - ByteSize: int64(thread.Arch().PtrSize()), - Name: "", - ReflectKind: reflect.Ptr, - Offset: 0, - }, - Type: typ, - } - } else { - name = "runtime.curg" - } - - return newVariableFromThread(thread, name, gaddr, typ), nil -} - -// GetG returns information on the G (goroutine) that is executing on this thread. -// -// The G structure for a thread is stored in thread local storage. Here we simply -// calculate the address and read and parse the G struct. -// -// We cannot simply use the allg linked list in order to find the M that represents -// the given OS thread and follow its G pointer because on Darwin mach ports are not -// universal, so our port for this thread would not map to the `id` attribute of the M -// structure. Also, when linked against libc, Go prefers the libc version of clone as -// opposed to the runtime version. This has the consequence of not setting M.id for -// any thread, regardless of OS. -// -// In order to get around all this craziness, we read the address of the G structure for -// the current thread from the thread local storage area. -func GetG(thread Thread) (*G, error) { - if thread.Common().g != nil { - return thread.Common().g, nil - } - if loc, _ := thread.Location(); loc != nil && loc.Fn != nil && loc.Fn.Name == "runtime.clone" { - // When threads are executing runtime.clone the value of TLS is unreliable. - return nil, nil - } - gaddr, err := getGVariable(thread) - if err != nil { - return nil, err - } - - g, err := gaddr.parseG() - if err != nil { - return nil, err - } - if g.ID == 0 { - // The runtime uses a special goroutine with ID == 0 to mark that the - // current goroutine is executing on the system stack (sometimes also - // referred to as the g0 stack or scheduler stack, I'm not sure if there's - // actually any difference between those). - // For our purposes it's better if we always return the real goroutine - // since the rest of the code assumes the goroutine ID is univocal. - // The real 'current goroutine' is stored in g0.m.curg - mvar, err := g.variable.structMember("m") - if err != nil { - return nil, err - } - curgvar, err := mvar.structMember("curg") - if err != nil { - return nil, err - } - g, err = curgvar.parseG() - if err != nil { - if _, ok := err.(ErrNoGoroutine); ok { - err = ErrNoGoroutine{thread.ThreadID()} - } - return nil, err - } - g.SystemStack = true - } - g.Thread = thread - if loc, err := thread.Location(); err == nil { - g.CurrentLoc = *loc - } - thread.Common().g = g - return g, nil -} - -// ThreadScope returns an EvalScope for this thread. -func ThreadScope(thread Thread) (*EvalScope, error) { - locations, err := ThreadStacktrace(thread, 1) - if err != nil { - return nil, err - } - if len(locations) < 1 { - return nil, errors.New("could not decode first frame") - } - return FrameToScope(thread.BinInfo(), thread, nil, locations...), nil -} - -// GoroutineScope returns an EvalScope for the goroutine running on this thread. -func GoroutineScope(thread Thread) (*EvalScope, error) { - locations, err := ThreadStacktrace(thread, 1) - if err != nil { - return nil, err - } - if len(locations) < 1 { - return nil, errors.New("could not decode first frame") - } - g, err := GetG(thread) - if err != nil { - return nil, err - } - return FrameToScope(thread.BinInfo(), thread, g, locations...), nil -} - -// onNextGoroutine returns true if this thread is on the goroutine requested by the current 'next' command -func onNextGoroutine(thread Thread, breakpoints *BreakpointMap) (bool, error) { - var bp *Breakpoint - for i := range breakpoints.M { - if breakpoints.M[i].Kind != UserBreakpoint && breakpoints.M[i].internalCond != nil { - bp = breakpoints.M[i] - break - } - } - if bp == nil { - return false, nil - } - // Internal breakpoint conditions can take multiple different forms: - // Step into breakpoints: - // runtime.curg.goid == X - // Next or StepOut breakpoints: - // runtime.curg.goid == X && runtime.frameoff == Y - // Breakpoints that can be hit either by stepping on a line in the same - // function or by returning from the function: - // runtime.curg.goid == X && (runtime.frameoff == Y || runtime.frameoff == Z) - // Here we are only interested in testing the runtime.curg.goid clause. - w := onNextGoroutineWalker{thread: thread} - ast.Walk(&w, bp.internalCond) - return w.ret, w.err -} - -type onNextGoroutineWalker struct { - thread Thread - ret bool - err error -} - -func (w *onNextGoroutineWalker) Visit(n ast.Node) ast.Visitor { - if binx, isbin := n.(*ast.BinaryExpr); isbin && binx.Op == token.EQL && exprToString(binx.X) == "runtime.curg.goid" { - w.ret, w.err = evalBreakpointCondition(w.thread, n.(ast.Expr)) - return nil - } - return w -} diff --git a/pkg/proc/variables.go b/pkg/proc/variables.go index 9b800f1c..073306b0 100644 --- a/pkg/proc/variables.go +++ b/pkg/proc/variables.go @@ -213,6 +213,253 @@ type G struct { labels *map[string]string // G's pprof labels, computed on demand in Labels() method } +// GetG returns information on the G (goroutine) that is executing on this thread. +// +// The G structure for a thread is stored in thread local storage. Here we simply +// calculate the address and read and parse the G struct. +// +// We cannot simply use the allg linked list in order to find the M that represents +// the given OS thread and follow its G pointer because on Darwin mach ports are not +// universal, so our port for this thread would not map to the `id` attribute of the M +// structure. Also, when linked against libc, Go prefers the libc version of clone as +// opposed to the runtime version. This has the consequence of not setting M.id for +// any thread, regardless of OS. +// +// In order to get around all this craziness, we read the address of the G structure for +// the current thread from the thread local storage area. +func GetG(thread Thread) (*G, error) { + if thread.Common().g != nil { + return thread.Common().g, nil + } + if loc, _ := thread.Location(); loc != nil && loc.Fn != nil && loc.Fn.Name == "runtime.clone" { + // When threads are executing runtime.clone the value of TLS is unreliable. + return nil, nil + } + gaddr, err := getGVariable(thread) + if err != nil { + return nil, err + } + + g, err := gaddr.parseG() + if err != nil { + return nil, err + } + if g.ID == 0 { + // The runtime uses a special goroutine with ID == 0 to mark that the + // current goroutine is executing on the system stack (sometimes also + // referred to as the g0 stack or scheduler stack, I'm not sure if there's + // actually any difference between those). + // For our purposes it's better if we always return the real goroutine + // since the rest of the code assumes the goroutine ID is univocal. + // The real 'current goroutine' is stored in g0.m.curg + mvar, err := g.variable.structMember("m") + if err != nil { + return nil, err + } + curgvar, err := mvar.structMember("curg") + if err != nil { + return nil, err + } + g, err = curgvar.parseG() + if err != nil { + if _, ok := err.(ErrNoGoroutine); ok { + err = ErrNoGoroutine{thread.ThreadID()} + } + return nil, err + } + g.SystemStack = true + } + g.Thread = thread + if loc, err := thread.Location(); err == nil { + g.CurrentLoc = *loc + } + thread.Common().g = g + return g, nil +} + +// GoroutinesInfo searches for goroutines starting at index 'start', and +// returns an array of up to 'count' (or all found elements, if 'count' is 0) +// G structures representing the information Delve care about from the internal +// runtime G structure. +// GoroutinesInfo also returns the next index to be used as 'start' argument +// while scanning for all available goroutines, or -1 if there was an error +// or if the index already reached the last possible value. +func GoroutinesInfo(dbp *Target, start, count int) ([]*G, int, error) { + if _, err := dbp.Valid(); err != nil { + return nil, -1, err + } + if dbp.gcache.allGCache != nil { + // We can't use the cached array to fulfill a subrange request + if start == 0 && (count == 0 || count >= len(dbp.gcache.allGCache)) { + return dbp.gcache.allGCache, -1, nil + } + } + + var ( + threadg = map[int]*G{} + allg []*G + ) + + threads := dbp.ThreadList() + for _, th := range threads { + if th.Blocked() { + continue + } + g, _ := GetG(th) + if g != nil { + threadg[g.ID] = g + } + } + + allgptr, allglen, err := dbp.gcache.getRuntimeAllg(dbp.BinInfo(), dbp.CurrentThread()) + if err != nil { + return nil, -1, err + } + + for i := uint64(start); i < allglen; i++ { + if count != 0 && len(allg) >= count { + return allg, int(i), nil + } + gvar, err := newGVariable(dbp.CurrentThread(), uintptr(allgptr+(i*uint64(dbp.BinInfo().Arch.PtrSize()))), true) + if err != nil { + allg = append(allg, &G{Unreadable: err}) + continue + } + g, err := gvar.parseG() + if err != nil { + allg = append(allg, &G{Unreadable: err}) + continue + } + if thg, allocated := threadg[g.ID]; allocated { + loc, err := thg.Thread.Location() + if err != nil { + return nil, -1, err + } + g.Thread = thg.Thread + // Prefer actual thread location information. + g.CurrentLoc = *loc + g.SystemStack = thg.SystemStack + } + if g.Status != Gdead { + allg = append(allg, g) + } + dbp.gcache.addGoroutine(g) + } + if start == 0 { + dbp.gcache.allGCache = allg + } + + return allg, -1, nil +} + +// FindGoroutine returns a G struct representing the goroutine +// specified by `gid`. +func FindGoroutine(dbp *Target, gid int) (*G, error) { + if selg := dbp.SelectedGoroutine(); (gid == -1) || (selg != nil && selg.ID == gid) || (selg == nil && gid == 0) { + // Return the currently selected goroutine in the following circumstances: + // + // 1. if the caller asks for gid == -1 (because that's what a goroutine ID of -1 means in our API). + // 2. if gid == selg.ID. + // this serves two purposes: (a) it's an optimizations that allows us + // to avoid reading any other goroutine and, more importantly, (b) we + // could be reading an incorrect value for the goroutine ID of a thread. + // This condition usually happens when a goroutine calls runtime.clone + // and for a short period of time two threads will appear to be running + // the same goroutine. + // 3. if the caller asks for gid == 0 and the selected goroutine is + // either 0 or nil. + // Goroutine 0 is special, it either means we have no current goroutine + // (for example, running C code), or that we are running on a speical + // stack (system stack, signal handling stack) and we didn't properly + // detect it. + // Since there could be multiple goroutines '0' running simultaneously + // if the user requests it return the one that's already selected or + // nil if there isn't a selected goroutine. + return selg, nil + } + + if gid == 0 { + return nil, fmt.Errorf("unknown goroutine %d", gid) + } + + // Calling GoroutinesInfo could be slow if there are many goroutines + // running, check if a running goroutine has been requested first. + for _, thread := range dbp.ThreadList() { + g, _ := GetG(thread) + if g != nil && g.ID == gid { + return g, nil + } + } + + if g := dbp.gcache.partialGCache[gid]; g != nil { + return g, nil + } + + const goroutinesInfoLimit = 10 + nextg := 0 + for nextg >= 0 { + var gs []*G + var err error + gs, nextg, err = GoroutinesInfo(dbp, nextg, goroutinesInfoLimit) + if err != nil { + return nil, err + } + for i := range gs { + if gs[i].ID == gid { + if gs[i].Unreadable != nil { + return nil, gs[i].Unreadable + } + return gs[i], nil + } + } + } + + return nil, fmt.Errorf("unknown goroutine %d", gid) +} + +func getGVariable(thread Thread) (*Variable, error) { + regs, err := thread.Registers(false) + if err != nil { + return nil, err + } + + gaddr, hasgaddr := regs.GAddr() + if !hasgaddr { + var err error + gaddr, err = readUintRaw(thread, uintptr(regs.TLS()+thread.BinInfo().GStructOffset()), int64(thread.BinInfo().Arch.PtrSize())) + if err != nil { + return nil, err + } + } + + return newGVariable(thread, uintptr(gaddr), thread.Arch().DerefTLS()) +} + +func newGVariable(thread Thread, gaddr uintptr, deref bool) (*Variable, error) { + typ, err := thread.BinInfo().findType("runtime.g") + if err != nil { + return nil, err + } + + name := "" + + if deref { + typ = &godwarf.PtrType{ + CommonType: godwarf.CommonType{ + ByteSize: int64(thread.Arch().PtrSize()), + Name: "", + ReflectKind: reflect.Ptr, + Offset: 0, + }, + Type: typ, + } + } else { + name = "runtime.curg" + } + + return newVariableFromThread(thread, name, gaddr, typ), nil +} + // Defer returns the top-most defer of the goroutine. func (g *G) Defer() *Defer { if g.variable.Unreadable != nil { @@ -427,7 +674,7 @@ func newVariable(name string, addr uintptr, dwarfType godwarf.Type, bi *BinaryIn case *godwarf.UnspecifiedType: v.Kind = reflect.Invalid default: - v.Unreadable = fmt.Errorf("Unknown type: %T", t) + v.Unreadable = fmt.Errorf("unknown type: %T", t) } return v diff --git a/pkg/terminal/command_test.go b/pkg/terminal/command_test.go index d7a9fd1f..448b8463 100644 --- a/pkg/terminal/command_test.go +++ b/pkg/terminal/command_test.go @@ -389,7 +389,7 @@ func TestScopePrefix(t *testing.T) { term.AssertExecError("frame", "not enough arguments") term.AssertExecError(fmt.Sprintf("goroutine %d frame 10 locals", curgid), fmt.Sprintf("Frame 10 does not exist in goroutine %d", curgid)) - term.AssertExecError("goroutine 9000 locals", "Unknown goroutine 9000") + term.AssertExecError("goroutine 9000 locals", "unknown goroutine 9000") term.AssertExecError("print n", "could not find symbol value for n") term.AssertExec("frame 1 print n", "3\n")