proc,service,terminal: read defer list

Adds -defer flag to the stack command that decorates the stack traces
by associating each stack frame with its deferred calls.

Reworks proc.next to use this feature instead of using proc.DeferPC,
laying the groundwork to implement #1240.

_fixtures/deferstack.go

@@ -0,0 +1,32 @@
+package main
+
+import "runtime"
+
+func f1() {
+}
+
+func f2() {
+}
+
+func f3() {
+}
+
+func call1() {
+	defer f2()
+	defer f1()
+	call2()
+}
+
+func call2() {
+	defer f3()
+	defer f2()
+	call3()
+}
+
+func call3() {
+	runtime.Breakpoint()
+}
+
+func main() {
+	call1()
+}

pkg/proc/core/core_test.go

@@ -185,7 +185,7 @@ func TestCore(t *testing.T) {
 	var panickingStack []proc.Stackframe
 	for _, g := range gs {
 		t.Logf("Goroutine %d", g.ID)
-		stack, err := g.Stacktrace(10)
+		stack, err := g.Stacktrace(10, false)
 		if err != nil {
 			t.Errorf("Stacktrace() on goroutine %v = %v", g, err)
 		}
@@ -329,7 +329,7 @@ func TestCoreWithEmptyString(t *testing.T) {
 	var mainFrame *proc.Stackframe
 mainSearch:
 	for _, g := range gs {
-		stack, err := g.Stacktrace(10)
+		stack, err := g.Stacktrace(10, false)
 		assertNoError(err, t, "Stacktrace()")
 		for _, frame := range stack {
 			if frame.Current.Fn != nil && frame.Current.Fn.Name == "main.main" {

pkg/proc/proc.go

@@ -380,14 +380,11 @@ func StepOut(dbp Process) error {
 
 	var deferpc uint64 = 0
 	if filepath.Ext(topframe.Current.File) == ".go" {
-		if selg != nil {
+		if topframe.TopmostDefer != nil && topframe.TopmostDefer.DeferredPC != 0 {
-			deferPCEntry := selg.DeferPC()
+			deferfn := dbp.BinInfo().PCToFunc(topframe.TopmostDefer.DeferredPC)
-			if deferPCEntry != 0 {
+			deferpc, err = FirstPCAfterPrologue(dbp, deferfn, false)
-				deferfn := dbp.BinInfo().PCToFunc(deferPCEntry)
+			if err != nil {
-				deferpc, err = FirstPCAfterPrologue(dbp, deferfn, false)
+				return err
-				if err != nil {
-					return err
-				}
 			}
 		}
 	}
@@ -554,7 +551,7 @@ func ConvertEvalScope(dbp Process, gid, frame int) (*EvalScope, error) {
 		thread = g.Thread
 	}
 
-	locs, err := g.Stacktrace(frame + 1)
+	locs, err := g.Stacktrace(frame+1, false)
 	if err != nil {
 		return nil, err
 	}

pkg/proc/proc_test.go

@@ -920,7 +920,7 @@ func TestStacktraceGoroutine(t *testing.T) {
 		mainCount := 0
 
 		for i, g := range gs {
-			locations, err := g.Stacktrace(40)
+			locations, err := g.Stacktrace(40, false)
 			if err != nil {
 				// On windows we do not have frame information for goroutines doing system calls.
 				t.Logf("Could not retrieve goroutine stack for goid=%d: %v", g.ID, err)
@@ -1237,13 +1237,13 @@ func TestFrameEvaluation(t *testing.T) {
 		found := make([]bool, 10)
 		for _, g := range gs {
 			frame := -1
-			frames, err := g.Stacktrace(10)
+			frames, err := g.Stacktrace(10, false)
 			if err != nil {
 				t.Logf("could not stacktrace goroutine %d: %v\n", g.ID, err)
 				continue
 			}
 			t.Logf("Goroutine %d", g.ID)
-			logStacktrace(t, frames)
+			logStacktrace(t, p.BinInfo(), frames)
 			for i := range frames {
 				if frames[i].Call.Fn != nil && frames[i].Call.Fn.Name == "main.agoroutine" {
 					frame = i
@@ -1957,7 +1957,7 @@ func TestNextParked(t *testing.T) {
 				if g.Thread != nil {
 					continue
 				}
-				frames, _ := g.Stacktrace(5)
+				frames, _ := g.Stacktrace(5, false)
 				for _, frame := range frames {
 					// line 11 is the line where wg.Done is called
 					if frame.Current.Fn != nil && frame.Current.Fn.Name == "main.sayhi" && frame.Current.Line < 11 {
@@ -2010,7 +2010,7 @@ func TestStepParked(t *testing.T) {
 		}
 
 		t.Logf("Parked g is: %v\n", parkedg)
-		frames, _ := parkedg.Stacktrace(20)
+		frames, _ := parkedg.Stacktrace(20, false)
 		for _, frame := range frames {
 			name := ""
 			if frame.Call.Fn != nil {
@@ -2714,7 +2714,7 @@ func TestStacktraceWithBarriers(t *testing.T) {
 				goid, _ := constant.Int64Val(goidVar.Value)
 
 				if g := getg(int(goid), gs); g != nil {
-					stack, err := g.Stacktrace(50)
+					stack, err := g.Stacktrace(50, false)
 					assertNoError(err, t, fmt.Sprintf("Stacktrace(goroutine = %d)", goid))
 					for _, frame := range stack {
 						if frame.Current.Fn != nil && frame.Current.Fn.Name == "main.bottomUpTree" {
@@ -2740,7 +2740,7 @@ func TestStacktraceWithBarriers(t *testing.T) {
 		for _, goid := range stackBarrierGoids {
 			g := getg(goid, gs)
 
-			stack, err := g.Stacktrace(200)
+			stack, err := g.Stacktrace(200, false)
 			assertNoError(err, t, "Stacktrace()")
 
 			// Check that either main.main or main.main.func1 appear in the
@@ -3142,7 +3142,7 @@ func TestIssue844(t *testing.T) {
 	})
 }
 
-func logStacktrace(t *testing.T, frames []proc.Stackframe) {
+func logStacktrace(t *testing.T, bi *proc.BinaryInfo, frames []proc.Stackframe) {
 	for j := range frames {
 		name := "?"
 		if frames[j].Current.Fn != nil {
@@ -3150,6 +3150,23 @@ func logStacktrace(t *testing.T, frames []proc.Stackframe) {
 		}
 
 		t.Logf("\t%#x %#x %#x %s at %s:%d\n", frames[j].Call.PC, frames[j].FrameOffset(), frames[j].FramePointerOffset(), name, filepath.Base(frames[j].Call.File), frames[j].Call.Line)
+		if frames[j].TopmostDefer != nil {
+			f, l, fn := bi.PCToLine(frames[j].TopmostDefer.DeferredPC)
+			fnname := ""
+			if fn != nil {
+				fnname = fn.Name
+			}
+			t.Logf("\t\ttopmost defer: %#x %s at %s:%d\n", frames[j].TopmostDefer.DeferredPC, fnname, f, l)
+		}
+		for deferIdx, _defer := range frames[j].Defers {
+			f, l, fn := bi.PCToLine(_defer.DeferredPC)
+			fnname := ""
+			if fn != nil {
+				fnname = fn.Name
+			}
+			t.Logf("\t\t%d defer: %#x %s at %s:%d\n", deferIdx, _defer.DeferredPC, fnname, f, l)
+
+		}
 	}
 }
 
@@ -3260,11 +3277,11 @@ func TestCgoStacktrace(t *testing.T) {
 				}
 			}
 
-			frames, err := g.Stacktrace(100)
+			frames, err := g.Stacktrace(100, false)
 			assertNoError(err, t, fmt.Sprintf("Stacktrace at iteration step %d", itidx))
 
 			t.Logf("iteration step %d", itidx)
-			logStacktrace(t, frames)
+			logStacktrace(t, p.BinInfo(), frames)
 
 			m := stacktraceCheck(t, tc, frames)
 			mismatch := (m == nil)
@@ -3301,7 +3318,7 @@ func TestCgoStacktrace(t *testing.T) {
 					if frames[j].Current.File != threadFrames[j].Current.File || frames[j].Current.Line != threadFrames[j].Current.Line {
 						t.Logf("stack mismatch between goroutine stacktrace and thread stacktrace")
 						t.Logf("thread stacktrace:")
-						logStacktrace(t, threadFrames)
+						logStacktrace(t, p.BinInfo(), threadFrames)
 						mismatch = true
 						break
 					}
@@ -3348,9 +3365,9 @@ func TestSystemstackStacktrace(t *testing.T) {
 		assertNoError(proc.Continue(p), t, "second continue")
 		g, err := proc.GetG(p.CurrentThread())
 		assertNoError(err, t, "GetG")
-		frames, err := g.Stacktrace(100)
+		frames, err := g.Stacktrace(100, false)
 		assertNoError(err, t, "stacktrace")
-		logStacktrace(t, frames)
+		logStacktrace(t, p.BinInfo(), frames)
 		m := stacktraceCheck(t, []string{"!runtime.startpanic_m", "runtime.gopanic", "main.main"}, frames)
 		if m == nil {
 			t.Fatal("see previous loglines")
@@ -3383,9 +3400,9 @@ func TestSystemstackOnRuntimeNewstack(t *testing.T) {
 				break
 			}
 		}
-		frames, err := g.Stacktrace(100)
+		frames, err := g.Stacktrace(100, false)
 		assertNoError(err, t, "stacktrace")
-		logStacktrace(t, frames)
+		logStacktrace(t, p.BinInfo(), frames)
 		m := stacktraceCheck(t, []string{"!runtime.newstack", "main.main"}, frames)
 		if m == nil {
 			t.Fatal("see previous loglines")
@@ -3400,7 +3417,7 @@ func TestIssue1034(t *testing.T) {
 		_, err := setFunctionBreakpoint(p, "main.main")
 		assertNoError(err, t, "setFunctionBreakpoint()")
 		assertNoError(proc.Continue(p), t, "Continue()")
-		frames, err := p.SelectedGoroutine().Stacktrace(10)
+		frames, err := p.SelectedGoroutine().Stacktrace(10, false)
 		assertNoError(err, t, "Stacktrace")
 		scope := proc.FrameToScope(p.BinInfo(), p.CurrentThread(), nil, frames[2:]...)
 		args, _ := scope.FunctionArguments(normalLoadConfig)
@@ -3873,3 +3890,62 @@ func TestIssue1264(t *testing.T) {
 		assertLineNumber(p, t, 8, "after continue")
 	})
 }
+
+func TestReadDefer(t *testing.T) {
+	withTestProcess("deferstack", t, func(p proc.Process, fixture protest.Fixture) {
+		assertNoError(proc.Continue(p), t, "Continue")
+		frames, err := p.SelectedGoroutine().Stacktrace(10, true)
+		assertNoError(err, t, "Stacktrace")
+
+		logStacktrace(t, p.BinInfo(), frames)
+
+		examples := []struct {
+			frameIdx     int
+			topmostDefer string
+			defers       []string
+		}{
+			// main.call3 (defers nothing, topmost defer main.f2)
+			{0, "main.f2", []string{}},
+
+			// main.call2 (defers main.f2, main.f3, topmost defer main.f2)
+			{1, "main.f2", []string{"main.f2", "main.f3"}},
+
+			// main.call1 (defers main.f1, main.f2, topmost defer main.f1)
+			{2, "main.f1", []string{"main.f1", "main.f2"}},
+
+			// main.main (defers nothing)
+			{3, "", []string{}}}
+
+		defercheck := func(d *proc.Defer, deferName, tgt string, frameIdx int) {
+			if d == nil {
+				t.Fatalf("expected %q as %s of frame %d, got nothing", tgt, deferName, frameIdx)
+			}
+			if d.Unreadable != nil {
+				t.Fatalf("expected %q as %s of frame %d, got unreadable defer: %v", tgt, deferName, frameIdx, d.Unreadable)
+			}
+			_, _, dfn := p.BinInfo().PCToLine(d.DeferredPC)
+			if dfn == nil {
+				t.Fatalf("expected %q as %s of frame %d, got %#x", tgt, deferName, frameIdx, d.DeferredPC)
+			}
+			if dfn.Name != tgt {
+				t.Fatalf("expected %q as %s of frame %d, got %q", tgt, deferName, frameIdx, dfn.Name)
+			}
+		}
+
+		for _, example := range examples {
+			frame := &frames[example.frameIdx]
+
+			if example.topmostDefer != "" {
+				defercheck(frame.TopmostDefer, "topmost defer", example.topmostDefer, example.frameIdx)
+			}
+
+			if len(example.defers) != len(frames[example.frameIdx].Defers) {
+				t.Fatalf("expected %d defers for %d, got %v", len(example.defers), example.frameIdx, frame.Defers)
+			}
+
+			for deferIdx := range example.defers {
+				defercheck(frame.Defers[deferIdx], fmt.Sprintf("defer %d", deferIdx), example.defers[deferIdx], example.frameIdx)
+			}
+		}
+	})
+}

pkg/proc/stack.go

@@ -4,6 +4,7 @@ import (
 	"debug/dwarf"
 	"errors"
 	"fmt"
+	"go/constant"
 	"strings"
 
 	"github.com/derekparker/delve/pkg/dwarf/frame"
@@ -57,6 +58,15 @@ type Stackframe struct {
 	// pkg/proc.
 	// Use this value to determine active lexical scopes for the stackframe.
 	lastpc uint64
+
+	// TopmostDefer is the defer that would be at the top of the stack when a
+	// panic unwind would get to this call frame, in other words it's the first
+	// deferred function that will  be called if the runtime unwinds past this
+	// call frame.
+	TopmostDefer *Defer
+
+	// Defers is the list of functions deferred by this stack frame (so far).
+	Defers []*Defer
 }
 
 // FrameOffset returns the address of the stack frame, absolute for system
@@ -91,7 +101,7 @@ func ThreadStacktrace(thread Thread, depth int) ([]Stackframe, error) {
 		it := newStackIterator(thread.BinInfo(), thread, thread.BinInfo().Arch.RegistersToDwarfRegisters(regs), 0, nil, -1, nil)
 		return it.stacktrace(depth)
 	}
-	return g.Stacktrace(depth)
+	return g.Stacktrace(depth, false)
 }
 
 func (g *G) stackIterator() (*stackIterator, error) {
@@ -112,12 +122,19 @@ func (g *G) stackIterator() (*stackIterator, error) {
 
 // Stacktrace returns the stack trace for a goroutine.
 // Note the locations in the array are return addresses not call addresses.
-func (g *G) Stacktrace(depth int) ([]Stackframe, error) {
+func (g *G) Stacktrace(depth int, readDefers bool) ([]Stackframe, error) {
 	it, err := g.stackIterator()
 	if err != nil {
 		return nil, err
 	}
-	return it.stacktrace(depth)
+	frames, err := it.stacktrace(depth)
+	if err != nil {
+		return nil, err
+	}
+	if readDefers {
+		g.readDefers(frames)
+	}
+	return frames, nil
 }
 
 // NullAddrError is an error for a null address.
@@ -568,3 +585,101 @@ func (it *stackIterator) readRegisterAt(regnum uint64, addr uint64) (*op.DwarfRe
 	}
 	return op.DwarfRegisterFromBytes(buf), nil
 }
+
+// Defer represents one deferred call
+type Defer struct {
+	DeferredPC uint64 // Value of field _defer.fn.fn, the deferred function
+	DeferPC    uint64 // PC address of instruction that added this defer
+	SP         uint64 // Value of SP register when this function was deferred (this field gets adjusted when the stack is moved to match the new stack space)
+	link       *Defer // Next deferred function
+
+	variable   *Variable
+	Unreadable error
+}
+
+// readDefers decorates the frames with the function deferred at each stack frame.
+func (g *G) readDefers(frames []Stackframe) {
+	curdefer := g.Defer()
+	i := 0
+
+	// scan simultaneously frames and the curdefer linked list, assigning
+	// defers to their associated frames.
+	for {
+		if curdefer == nil || i >= len(frames) {
+			return
+		}
+		if curdefer.Unreadable != nil {
+			// Current defer is unreadable, stick it into the first available frame
+			// (so that it can be reported to the user) and exit
+			frames[i].Defers = append(frames[i].Defers, curdefer)
+			return
+		}
+		if frames[i].Err != nil {
+			return
+		}
+
+		if frames[i].TopmostDefer == nil {
+			frames[i].TopmostDefer = curdefer
+		}
+
+		if frames[i].SystemStack || curdefer.SP >= uint64(frames[i].Regs.CFA) {
+			// frames[i].Regs.CFA is the value that SP had before the function of
+			// frames[i] was called.
+			// This means that when curdefer.SP == frames[i].Regs.CFA then curdefer
+			// was added by the previous frame.
+			//
+			// curdefer.SP < frames[i].Regs.CFA means curdefer was added by a
+			// function further down the stack.
+			//
+			// SystemStack frames live on a different physical stack and can't be
+			// compared with deferred frames.
+			i++
+		} else {
+			frames[i].Defers = append(frames[i].Defers, curdefer)
+			curdefer = curdefer.Next()
+		}
+	}
+}
+
+func (d *Defer) load() {
+	d.variable.loadValue(LoadConfig{false, 1, 0, 0, -1})
+	if d.variable.Unreadable != nil {
+		d.Unreadable = d.variable.Unreadable
+		return
+	}
+
+	fnvar := d.variable.fieldVariable("fn").maybeDereference()
+	if fnvar.Addr != 0 {
+		fnvar = fnvar.loadFieldNamed("fn")
+		if fnvar.Unreadable == nil {
+			d.DeferredPC, _ = constant.Uint64Val(fnvar.Value)
+		}
+	}
+
+	d.DeferPC, _ = constant.Uint64Val(d.variable.fieldVariable("pc").Value)
+	d.SP, _ = constant.Uint64Val(d.variable.fieldVariable("sp").Value)
+
+	linkvar := d.variable.fieldVariable("link").maybeDereference()
+	if linkvar.Addr != 0 {
+		d.link = &Defer{variable: linkvar}
+	}
+}
+
+// spDecreasedErr is used when (*Defer).Next detects a corrupted linked
+// list, specifically when after followin a link pointer the value of SP
+// decreases rather than increasing or staying the same (the defer list is a
+// FIFO list, nodes further down the list have been added by function calls
+// further down the call stack and therefore the SP should always increase).
+var spDecreasedErr = errors.New("corrupted defer list: SP decreased")
+
+// Next returns the next defer in the linked list
+func (d *Defer) Next() *Defer {
+	if d.link == nil {
+		return nil
+	}
+	d.link.load()
+	if d.link.SP < d.SP {
+		d.link.Unreadable = spDecreasedErr
+	}
+	return d.link
+}

pkg/proc/threads.go

@@ -79,7 +79,7 @@ func topframe(g *G, thread Thread) (Stackframe, Stackframe, error) {
 		}
 		frames, err = ThreadStacktrace(thread, 1)
 	} else {
-		frames, err = g.Stacktrace(1)
+		frames, err = g.Stacktrace(1, true)
 	}
 	if err != nil {
 		return Stackframe{}, Stackframe{}, err
@@ -224,15 +224,12 @@ func next(dbp Process, stepInto, inlinedStepOut bool) error {
 
 		// Set breakpoint on the most recently deferred function (if any)
 		var deferpc uint64 = 0
-		if selg != nil {
+		if topframe.TopmostDefer != nil && topframe.TopmostDefer.DeferredPC != 0 {
-			deferPCEntry := selg.DeferPC()
+			deferfn := dbp.BinInfo().PCToFunc(topframe.TopmostDefer.DeferredPC)
-			if deferPCEntry != 0 {
+			var err error
-				deferfn := dbp.BinInfo().PCToFunc(deferPCEntry)
+			deferpc, err = FirstPCAfterPrologue(dbp, deferfn, false)
-				var err error
+			if err != nil {
-				deferpc, err = FirstPCAfterPrologue(dbp, deferfn, false)
+				return err
-				if err != nil {
-					return err
-				}
 			}
 		}
 		if deferpc != 0 && deferpc != topframe.Current.PC {

pkg/proc/variables.go

@@ -489,29 +489,18 @@ func (v *Variable) fieldVariable(name string) *Variable {
 	return nil
 }
 
-// PC of entry to top-most deferred function.
+// Defer returns the top-most defer of the goroutine.
-func (g *G) DeferPC() uint64 {
+func (g *G) Defer() *Defer {
 	if g.variable.Unreadable != nil {
-		return 0
+		return nil
 	}
-	d := g.variable.fieldVariable("_defer").maybeDereference()
+	dvar := g.variable.fieldVariable("_defer").maybeDereference()
-	if d.Addr == 0 {
+	if dvar.Addr == 0 {
-		return 0
+		return nil
 	}
-	d.loadValue(LoadConfig{false, 1, 64, 0, -1})
+	d := &Defer{variable: dvar}
-	if d.Unreadable != nil {
+	d.load()
-		return 0
+	return d
-	}
-	fnvar := d.fieldVariable("fn").maybeDereference()
-	if fnvar.Addr == 0 {
-		return 0
-	}
-	fnvar.loadValue(LoadConfig{false, 1, 64, 0, -1})
-	if fnvar.Unreadable != nil {
-		return 0
-	}
-	deferPC, _ := constant.Int64Val(fnvar.fieldVariable("fn").Value)
-	return uint64(deferPC)
 }
 
 // From $GOROOT/src/runtime/traceback.go:597

pkg/terminal/command.go

@@ -503,7 +503,7 @@ func threads(t *Term, ctx callContext, args string) error {
 		if th.Function != nil {
 			fmt.Printf("%sThread %d at %#v %s:%d %s\n",
 				prefix, th.ID, th.PC, ShortenFilePath(th.File),
-				th.Line, th.Function.Name)
+				th.Line, th.Function.Name())
 		} else {
 			fmt.Printf("%sThread %s\n", prefix, formatThread(th))
 		}
@@ -593,7 +593,7 @@ func goroutines(t *Term, ctx callContext, argstr string) error {
 		}
 		fmt.Printf("%sGoroutine %s\n", prefix, formatGoroutine(g, fgl))
 		if bPrintStack {
-			stack, err := t.client.Stacktrace(g.ID, 10, nil)
+			stack, err := t.client.Stacktrace(g.ID, 10, false, nil)
 			if err != nil {
 				return err
 			}
@@ -682,7 +682,7 @@ func (c *Commands) frameCommand(t *Term, ctx callContext, argstr string, directi
 	if frame < 0 {
 		return fmt.Errorf("Invalid frame %d", frame)
 	}
-	stack, err := t.client.Stacktrace(ctx.Scope.GoroutineID, frame, nil)
+	stack, err := t.client.Stacktrace(ctx.Scope.GoroutineID, frame, false, nil)
 	if err != nil {
 		return err
 	}
@@ -731,11 +731,7 @@ const (
 )
 
 func formatLocation(loc api.Location) string {
-	fname := ""
+	return fmt.Sprintf("%s:%d %s (%#v)", ShortenFilePath(loc.File), loc.Line, loc.Function.Name(), loc.PC)
-	if loc.Function != nil {
-		fname = loc.Function.Name
-	}
-	return fmt.Sprintf("%s:%d %s (%#v)", ShortenFilePath(loc.File), loc.Line, fname, loc.PC)
 }
 
 func formatGoroutine(g *api.Goroutine, fgl formatGoroutineLoc) string {
@@ -1320,7 +1316,7 @@ func stackCommand(t *Term, ctx callContext, args string) error {
 	if sa.full {
 		cfg = &ShortLoadConfig
 	}
-	stack, err := t.client.Stacktrace(ctx.Scope.GoroutineID, sa.depth, cfg)
+	stack, err := t.client.Stacktrace(ctx.Scope.GoroutineID, sa.depth, sa.readDefers, cfg)
 	if err != nil {
 		return err
 	}
@@ -1329,9 +1325,10 @@ func stackCommand(t *Term, ctx callContext, args string) error {
 }
 
 type stackArgs struct {
-	depth   int
+	depth      int
-	full    bool
+	full       bool
-	offsets bool
+	offsets    bool
+	readDefers bool
 }
 
 func parseStackArgs(argstr string) (stackArgs, error) {
@@ -1347,6 +1344,8 @@ func parseStackArgs(argstr string) (stackArgs, error) {
 				r.full = true
 			case "-offsets":
 				r.offsets = true
+			case "-defer":
+				r.readDefers = true
 			default:
 				n, err := strconv.Atoi(args[i])
 				if err != nil {
@@ -1373,7 +1372,7 @@ func listCommand(t *Term, ctx callContext, args string) error {
 		return printfile(t, state.CurrentThread.File, state.CurrentThread.Line, true)
 
 	case len(args) == 0 && ctx.scoped():
-		locs, err := t.client.Stacktrace(ctx.Scope.GoroutineID, ctx.Scope.Frame, nil)
+		locs, err := t.client.Stacktrace(ctx.Scope.GoroutineID, ctx.Scope.Frame, false, nil)
 		if err != nil {
 			return err
 		}
@@ -1487,6 +1486,15 @@ func printStack(stack []api.Stackframe, ind string, offsets bool) {
 	if len(stack) == 0 {
 		return
 	}
+
+	extranl := offsets
+	for i := range stack {
+		if extranl {
+			break
+		}
+		extranl = extranl || (len(stack[i].Defers) > 0) || (len(stack[i].Arguments) > 0) || (len(stack[i].Locals) > 0)
+	}
+
 	d := digits(len(stack) - 1)
 	fmtstr := "%s%" + strconv.Itoa(d) + "d  0x%016x in %s\n"
 	s := ind + strings.Repeat(" ", d+2+len(ind))
@@ -1496,23 +1504,35 @@ func printStack(stack []api.Stackframe, ind string, offsets bool) {
 			fmt.Printf("%serror: %s\n", s, stack[i].Err)
 			continue
 		}
-		name := "(nil)"
+		fmt.Printf(fmtstr, ind, i, stack[i].PC, stack[i].Function.Name())
-		if stack[i].Function != nil {
-			name = stack[i].Function.Name
-		}
-		fmt.Printf(fmtstr, ind, i, stack[i].PC, name)
 		fmt.Printf("%sat %s:%d\n", s, ShortenFilePath(stack[i].File), stack[i].Line)
 
 		if offsets {
 			fmt.Printf("%sframe: %+#x frame pointer %+#x\n", s, stack[i].FrameOffset, stack[i].FramePointerOffset)
 		}
 
+		for j, d := range stack[i].Defers {
+			deferHeader := fmt.Sprintf("%s    defer %d: ", s, j)
+			s2 := strings.Repeat(" ", len(deferHeader))
+			if d.Unreadable != "" {
+				fmt.Printf("%s(unreadable defer: %s)\n", deferHeader, d.Unreadable)
+				continue
+			}
+			fmt.Printf("%s%#016x in %s\n", deferHeader, d.DeferredLoc.PC, d.DeferredLoc.Function.Name())
+			fmt.Printf("%sat %s:%d\n", s2, d.DeferredLoc.File, d.DeferredLoc.Line)
+			fmt.Printf("%sdeferred by %s at %s:%d\n", s2, d.DeferLoc.Function.Name(), d.DeferLoc.File, d.DeferLoc.Line)
+		}
+
 		for j := range stack[i].Arguments {
 			fmt.Printf("%s    %s = %s\n", s, stack[i].Arguments[j].Name, stack[i].Arguments[j].SinglelineString())
 		}
 		for j := range stack[i].Locals {
 			fmt.Printf("%s    %s = %s\n", s, stack[i].Locals[j].Name, stack[i].Locals[j].SinglelineString())
 		}
+
+		if extranl {
+			fmt.Println()
+		}
 	}
 }
 
@@ -1563,7 +1583,7 @@ func printcontext(t *Term, state *api.DebuggerState) error {
 }
 
 func printcontextLocation(loc api.Location) {
-	fmt.Printf("> %s() %s:%d (PC: %#v)\n", loc.Function.Name, ShortenFilePath(loc.File), loc.Line, loc.PC)
+	fmt.Printf("> %s() %s:%d (PC: %#v)\n", loc.Function.Name(), ShortenFilePath(loc.File), loc.Line, loc.PC)
 	if loc.Function != nil && loc.Function.Optimized {
 		fmt.Println(optimizedFunctionWarning)
 	}
@@ -1607,7 +1627,7 @@ func printcontextThread(t *Term, th *api.Thread) {
 	if hitCount, ok := th.Breakpoint.HitCount[strconv.Itoa(th.GoroutineID)]; ok {
 		fmt.Printf("> %s%s(%s) %s:%d (hits goroutine(%d):%d total:%d) (PC: %#v)\n",
 			bpname,
-			fn.Name,
+			fn.Name(),
 			args,
 			ShortenFilePath(th.File),
 			th.Line,
@@ -1618,7 +1638,7 @@ func printcontextThread(t *Term, th *api.Thread) {
 	} else {
 		fmt.Printf("> %s%s(%s) %s:%d (hits total:%d) (PC: %#v)\n",
 			bpname,
-			fn.Name,
+			fn.Name(),
 			args,
 			ShortenFilePath(th.File),
 			th.Line,
@@ -1829,11 +1849,7 @@ func checkpoint(t *Term, ctx callContext, args string) error {
 		if state.SelectedGoroutine != nil {
 			loc = state.SelectedGoroutine.CurrentLoc
 		}
-		fname := "???"
+		args = fmt.Sprintf("%s() %s:%d (%#x)", loc.Function.Name(), loc.File, loc.Line, loc.PC)
-		if loc.Function != nil {
-			fname = loc.Function.Name
-		}
-		args = fmt.Sprintf("%s() %s:%d (%#x)", fname, loc.File, loc.Line, loc.PC)
 	}
 
 	cpid, err := t.client.Checkpoint(args)

pkg/terminal/command_test.go

@@ -224,7 +224,7 @@ func TestExecuteFile(t *testing.T) {
 
 func TestIssue354(t *testing.T) {
 	printStack([]api.Stackframe{}, "", false)
-	printStack([]api.Stackframe{{api.Location{PC: 0, File: "irrelevant.go", Line: 10, Function: nil}, nil, nil, 0, 0, ""}}, "", false)
+	printStack([]api.Stackframe{{api.Location{PC: 0, File: "irrelevant.go", Line: 10, Function: nil}, nil, nil, 0, 0, nil, ""}}, "", false)
 }
 
 func TestIssue411(t *testing.T) {

pkg/terminal/disasmprint.go

@@ -14,7 +14,7 @@ func DisasmPrint(dv api.AsmInstructions, out io.Writer) {
 	bw := bufio.NewWriter(out)
 	defer bw.Flush()
 	if len(dv) > 0 && dv[0].Loc.Function != nil {
-		fmt.Fprintf(bw, "TEXT %s(SB) %s\n", dv[0].Loc.Function.Name, dv[0].Loc.File)
+		fmt.Fprintf(bw, "TEXT %s(SB) %s\n", dv[0].Loc.Function.Name(), dv[0].Loc.File)
 	}
 	tw := tabwriter.NewWriter(bw, 1, 8, 1, '\t', 0)
 	defer tw.Flush()

service/api/conversions.go

@@ -215,7 +215,7 @@ func ConvertFunction(fn *proc.Function) *Function {
 	// those fields is not documented their value was replaced with 0 when
 	// gosym.Func was replaced by debug_info entries.
 	return &Function{
-		Name:      fn.Name,
+		Name_:     fn.Name,
 		Type:      0,
 		Value:     fn.Entry,
 		GoType:    0,

service/api/types.go

@@ -129,9 +129,18 @@ type Stackframe struct {
 	FrameOffset        int64
 	FramePointerOffset int64
 
+	Defers []Defer
+
 	Err string
 }
 
+type Defer struct {
+	DeferredLoc Location // deferred function
+	DeferLoc    Location // location of the defer statement
+	SP          uint64   // value of SP when the function was deferred
+	Unreadable  string
+}
+
 func (frame *Stackframe) Var(name string) *Variable {
 	for i := range frame.Locals {
 		if frame.Locals[i].Name == name {
@@ -149,7 +158,7 @@ func (frame *Stackframe) Var(name string) *Variable {
 // Function represents thread-scoped function information.
 type Function struct {
 	// Name is the function name.
-	Name   string `json:"name"`
+	Name_  string `json:"name"`
 	Value  uint64 `json:"value"`
 	Type   byte   `json:"type"`
 	GoType uint64 `json:"goType"`
@@ -157,6 +166,13 @@ type Function struct {
 	Optimized bool `json:"optimized"`
 }
 
+func (fn *Function) Name() string {
+	if fn == nil {
+		return "???"
+	}
+	return fn.Name_
+}
+
 // VariableFlags is the type of the Flags field of Variable.
 type VariableFlags uint16
 

service/client.go

@@ -98,7 +98,7 @@ type Client interface {
 	ListGoroutines() ([]*api.Goroutine, error)
 
 	// Returns stacktrace
-	Stacktrace(int, int, *api.LoadConfig) ([]api.Stackframe, error)
+	Stacktrace(goroutineID int, depth int, readDefers bool, cfg *api.LoadConfig) ([]api.Stackframe, error)
 
 	// Returns whether we attached to a running process or not
 	AttachedToExistingProcess() bool

service/debugger/debugger.go

@@ -895,7 +895,7 @@ func (d *Debugger) Goroutines() ([]*api.Goroutine, error) {
 // Stacktrace returns a list of Stackframes for the given goroutine. The
 // length of the returned list will be min(stack_len, depth).
 // If 'full' is true, then local vars, function args, etc will be returned as well.
-func (d *Debugger) Stacktrace(goroutineID, depth int, cfg *proc.LoadConfig) ([]api.Stackframe, error) {
+func (d *Debugger) Stacktrace(goroutineID, depth int, readDefers bool, cfg *proc.LoadConfig) ([]api.Stackframe, error) {
 	d.processMutex.Lock()
 	defer d.processMutex.Unlock()
 
@@ -913,7 +913,7 @@ func (d *Debugger) Stacktrace(goroutineID, depth int, cfg *proc.LoadConfig) ([]a
 	if g == nil {
 		rawlocs, err = proc.ThreadStacktrace(d.target.CurrentThread(), depth)
 	} else {
-		rawlocs, err = g.Stacktrace(depth)
+		rawlocs, err = g.Stacktrace(depth, readDefers)
 	}
 	if err != nil {
 		return nil, err
@@ -930,6 +930,8 @@ func (d *Debugger) convertStacktrace(rawlocs []proc.Stackframe, cfg *proc.LoadCo
 
 			FrameOffset:        rawlocs[i].FrameOffset(),
 			FramePointerOffset: rawlocs[i].FramePointerOffset(),
+
+			Defers: d.convertDefers(rawlocs[i].Defers),
 		}
 		if rawlocs[i].Err != nil {
 			frame.Err = rawlocs[i].Err.Error()
@@ -955,6 +957,36 @@ func (d *Debugger) convertStacktrace(rawlocs []proc.Stackframe, cfg *proc.LoadCo
 	return locations, nil
 }
 
+func (d *Debugger) convertDefers(defers []*proc.Defer) []api.Defer {
+	r := make([]api.Defer, len(defers))
+	for i := range defers {
+		ddf, ddl, ddfn := d.target.BinInfo().PCToLine(defers[i].DeferredPC)
+		drf, drl, drfn := d.target.BinInfo().PCToLine(defers[i].DeferPC)
+
+		r[i] = api.Defer{
+			DeferredLoc: api.ConvertLocation(proc.Location{
+				PC:   defers[i].DeferredPC,
+				File: ddf,
+				Line: ddl,
+				Fn:   ddfn,
+			}),
+			DeferLoc: api.ConvertLocation(proc.Location{
+				PC:   defers[i].DeferPC,
+				File: drf,
+				Line: drl,
+				Fn:   drfn,
+			}),
+			SP: defers[i].SP,
+		}
+
+		if defers[i].Unreadable != nil {
+			r[i].Unreadable = defers[i].Unreadable.Error()
+		}
+	}
+
+	return r
+}
+
 // FindLocation will find the location specified by 'locStr'.
 func (d *Debugger) FindLocation(scope api.EvalScope, locStr string) ([]api.Location, error) {
 	d.processMutex.Lock()

service/debugger/locations.go

@@ -316,7 +316,7 @@ func (ale AmbiguousLocationError) Error() string {
 	var candidates []string
 	if ale.CandidatesLocation != nil {
 		for i := range ale.CandidatesLocation {
-			candidates = append(candidates, ale.CandidatesLocation[i].Function.Name)
+			candidates = append(candidates, ale.CandidatesLocation[i].Function.Name())
 		}
 
 	} else {

service/rpc1/server.go

@@ -87,7 +87,7 @@ func (s *RPCServer) StacktraceGoroutine(args *StacktraceGoroutineArgs, locations
 	if args.Full {
 		loadcfg = &defaultLoadConfig
 	}
-	locs, err := s.debugger.Stacktrace(args.Id, args.Depth, loadcfg)
+	locs, err := s.debugger.Stacktrace(args.Id, args.Depth, false, loadcfg)
 	if err != nil {
 		return err
 	}

service/rpc2/client.go

@@ -295,9 +295,9 @@ func (c *RPCClient) ListGoroutines() ([]*api.Goroutine, error) {
 	return out.Goroutines, err
 }
 
-func (c *RPCClient) Stacktrace(goroutineId, depth int, cfg *api.LoadConfig) ([]api.Stackframe, error) {
+func (c *RPCClient) Stacktrace(goroutineId, depth int, readDefers bool, cfg *api.LoadConfig) ([]api.Stackframe, error) {
 	var out StacktraceOut
-	err := c.call("Stacktrace", StacktraceIn{goroutineId, depth, false, cfg}, &out)
+	err := c.call("Stacktrace", StacktraceIn{goroutineId, depth, false, readDefers, cfg}, &out)
 	return out.Locations, err
 }
 

service/rpc2/server.go

@@ -152,10 +152,11 @@ func (s *RPCServer) GetBreakpoint(arg GetBreakpointIn, out *GetBreakpointOut) er
 }
 
 type StacktraceIn struct {
-	Id    int
+	Id     int
-	Depth int
+	Depth  int
-	Full  bool
+	Full   bool
-	Cfg   *api.LoadConfig
+	Defers bool // read deferred functions
+	Cfg    *api.LoadConfig
 }
 
 type StacktraceOut struct {
@@ -171,7 +172,7 @@ func (s *RPCServer) Stacktrace(arg StacktraceIn, out *StacktraceOut) error {
 	if cfg == nil && arg.Full {
 		cfg = &api.LoadConfig{true, 1, 64, 64, -1}
 	}
-	locs, err := s.debugger.Stacktrace(arg.Id, arg.Depth, api.LoadConfigToProc(cfg))
+	locs, err := s.debugger.Stacktrace(arg.Id, arg.Depth, arg.Defers, api.LoadConfigToProc(cfg))
 	if err != nil {
 		return err
 	}

service/test/integration1_test.go

@@ -732,7 +732,7 @@ func Test1ClientServer_FullStacktrace(t *testing.T) {
 				if frame.Function == nil {
 					continue
 				}
-				if frame.Function.Name != "main.agoroutine" {
+				if frame.Function.Name() != "main.agoroutine" {
 					continue
 				}
 				t.Logf("frame %d: %v", i, frame)
@@ -887,7 +887,7 @@ func Test1Disasm(t *testing.T) {
 		// look for static call to afunction() on line 29
 		found := false
 		for i := range d3 {
-			if d3[i].Loc.Line == 29 && strings.HasPrefix(d3[i].Text, "call") && d3[i].DestLoc != nil && d3[i].DestLoc.Function != nil && d3[i].DestLoc.Function.Name == "main.afunction" {
+			if d3[i].Loc.Line == 29 && strings.HasPrefix(d3[i].Text, "call") && d3[i].DestLoc != nil && d3[i].DestLoc.Function != nil && d3[i].DestLoc.Function.Name() == "main.afunction" {
 				found = true
 				break
 			}
@@ -937,7 +937,7 @@ func Test1Disasm(t *testing.T) {
 				if curinstr.DestLoc == nil || curinstr.DestLoc.Function == nil {
 					t.Fatalf("Call instruction does not have destination: %v", curinstr)
 				}
-				if curinstr.DestLoc.Function.Name != "main.afunction" {
+				if curinstr.DestLoc.Function.Name() != "main.afunction" {
 					t.Fatalf("Call instruction destination not main.afunction: %v", curinstr)
 				}
 				break

service/test/integration2_test.go

@@ -798,13 +798,13 @@ func TestClientServer_FullStacktrace(t *testing.T) {
 		assertNoError(err, t, "GoroutinesInfo()")
 		found := make([]bool, 10)
 		for _, g := range gs {
-			frames, err := c.Stacktrace(g.ID, 10, &normalLoadConfig)
+			frames, err := c.Stacktrace(g.ID, 10, false, &normalLoadConfig)
 			assertNoError(err, t, fmt.Sprintf("Stacktrace(%d)", g.ID))
 			for i, frame := range frames {
 				if frame.Function == nil {
 					continue
 				}
-				if frame.Function.Name != "main.agoroutine" {
+				if frame.Function.Name() != "main.agoroutine" {
 					continue
 				}
 				t.Logf("frame %d: %v", i, frame)
@@ -832,7 +832,7 @@ func TestClientServer_FullStacktrace(t *testing.T) {
 			t.Fatalf("Continue(): %v\n", state.Err)
 		}
 
-		frames, err := c.Stacktrace(-1, 10, &normalLoadConfig)
+		frames, err := c.Stacktrace(-1, 10, false, &normalLoadConfig)
 		assertNoError(err, t, "Stacktrace")
 
 		cur := 3
@@ -911,7 +911,7 @@ func TestIssue355(t *testing.T) {
 		assertError(err, t, "ListRegisters()")
 		_, err = c.ListGoroutines()
 		assertError(err, t, "ListGoroutines()")
-		_, err = c.Stacktrace(gid, 10, &normalLoadConfig)
+		_, err = c.Stacktrace(gid, 10, false, &normalLoadConfig)
 		assertError(err, t, "Stacktrace()")
 		_, err = c.FindLocation(api.EvalScope{gid, 0}, "+1")
 		assertError(err, t, "FindLocation()")
@@ -964,7 +964,7 @@ func TestDisasm(t *testing.T) {
 		// look for static call to afunction() on line 29
 		found := false
 		for i := range d3 {
-			if d3[i].Loc.Line == 29 && strings.HasPrefix(d3[i].Text, "call") && d3[i].DestLoc != nil && d3[i].DestLoc.Function != nil && d3[i].DestLoc.Function.Name == "main.afunction" {
+			if d3[i].Loc.Line == 29 && strings.HasPrefix(d3[i].Text, "call") && d3[i].DestLoc != nil && d3[i].DestLoc.Function != nil && d3[i].DestLoc.Function.Name() == "main.afunction" {
 				found = true
 				break
 			}
@@ -1014,7 +1014,7 @@ func TestDisasm(t *testing.T) {
 				if curinstr.DestLoc == nil || curinstr.DestLoc.Function == nil {
 					t.Fatalf("Call instruction does not have destination: %v", curinstr)
 				}
-				if curinstr.DestLoc.Function.Name != "main.afunction" {
+				if curinstr.DestLoc.Function.Name() != "main.afunction" {
 					t.Fatalf("Call instruction destination not main.afunction: %v", curinstr)
 				}
 				break
@@ -1034,7 +1034,7 @@ func TestNegativeStackDepthBug(t *testing.T) {
 		ch := c.Continue()
 		state := <-ch
 		assertNoError(state.Err, t, "Continue()")
-		_, err = c.Stacktrace(-1, -2, &normalLoadConfig)
+		_, err = c.Stacktrace(-1, -2, false, &normalLoadConfig)
 		assertError(err, t, "Stacktrace()")
 	})
 }
@@ -1493,7 +1493,7 @@ func TestAcceptMulticlient(t *testing.T) {
 
 	client2 := rpc2.NewClient(listener.Addr().String())
 	state := <-client2.Continue()
-	if state.CurrentThread.Function.Name != "main.main" {
+	if state.CurrentThread.Function.Name() != "main.main" {
 		t.Fatalf("bad state after continue: %v\n", state)
 	}
 	client2.Detach(true)
@@ -1516,12 +1516,12 @@ func TestClientServerFunctionCall(t *testing.T) {
 		c.SetReturnValuesLoadConfig(&normalLoadConfig)
 		state := <-c.Continue()
 		assertNoError(state.Err, t, "Continue()")
-		beforeCallFn := state.CurrentThread.Function.Name
+		beforeCallFn := state.CurrentThread.Function.Name()
 		state, err := c.Call("call1(one, two)")
 		assertNoError(err, t, "Call()")
-		t.Logf("returned to %q", state.CurrentThread.Function.Name)
+		t.Logf("returned to %q", state.CurrentThread.Function.Name())
-		if state.CurrentThread.Function.Name != beforeCallFn {
+		if state.CurrentThread.Function.Name() != beforeCallFn {
-			t.Fatalf("did not return to the calling function %q %q", beforeCallFn, state.CurrentThread.Function.Name)
+			t.Fatalf("did not return to the calling function %q %q", beforeCallFn, state.CurrentThread.Function.Name())
 		}
 		if state.CurrentThread.ReturnValues == nil {
 			t.Fatal("no return values on return from call")