proc,debugger,terminal: read goroutine ancestors

Add options to the stack command to read the goroutine ancestors.
Ancestor tracking was added to Go 1.12 with CL:
https://go-review.googlesource.com/c/go/+/70993/

Implements #1491

Documentation/cli/README.md

@@ -357,11 +357,13 @@ If regex is specified only the source files matching it will be returned.
 ## stack
 Print stack trace.
 
-	[goroutine <n>] [frame <m>] stack [<depth>] [-full] [-offsets] [-defer]
+	[goroutine <n>] [frame <m>] stack [<depth>] [-full] [-offsets] [-defer] [-a <n>] [-adepth <depth>]
 
 	-full		every stackframe is decorated with the value of its local variables and arguments.
 	-offsets	prints frame offset of each frame.
 	-defer		prints deferred function call stack for each frame.
+	-a <n>		prints stacktrace of n ancestors of the selected goroutine (target process must have tracebackancestors enabled)
+	-adepth <depth>	configures depth of ancestor stacktrace
 
 
 Aliases: bt

pkg/proc/proc_test.go

@@ -4251,3 +4251,38 @@ func TestListImages(t *testing.T) {
 		}
 	})
 }
+
+func TestAncestors(t *testing.T) {
+	if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 11) {
+		t.Skip("not supported on Go <= 1.10")
+	}
+	savedGodebug := os.Getenv("GODEBUG")
+	os.Setenv("GODEBUG", "tracebackancestors=100")
+	defer os.Setenv("GODEBUG", savedGodebug)
+	withTestProcess("testnextprog", t, func(p proc.Process, fixture protest.Fixture) {
+		_, err := setFunctionBreakpoint(p, "main.testgoroutine")
+		assertNoError(err, t, "setFunctionBreakpoint()")
+		assertNoError(proc.Continue(p), t, "Continue()")
+		as, err := p.SelectedGoroutine().Ancestors(1000)
+		assertNoError(err, t, "Ancestors")
+		t.Logf("ancestors: %#v\n", as)
+		if len(as) != 1 {
+			t.Fatalf("expected only one ancestor got %d", len(as))
+		}
+		mainFound := false
+		for i, a := range as {
+			astack, err := a.Stack(100)
+			assertNoError(err, t, fmt.Sprintf("Ancestor %d stack", i))
+			t.Logf("ancestor %d\n", i)
+			logStacktrace(t, p.BinInfo(), astack)
+			for _, frame := range astack {
+				if frame.Current.Fn != nil && frame.Current.Fn.Name == "main.main" {
+					mainFound = true
+				}
+			}
+		}
+		if !mainFound {
+			t.Fatal("could not find main.main function in ancestors")
+		}
+	})
+}

pkg/proc/variables.go

@@ -200,6 +200,12 @@ type G struct {
 	Unreadable error // could not read the G struct
 }
 
+type Ancestor struct {
+	ID         int64 // Goroutine ID
+	Unreadable error
+	pcsVar     *Variable
+}
+
 // EvalScope is the scope for variable evaluation. Contains the thread,
 // current location (PC), and canonical frame address.
 type EvalScope struct {
@@ -617,6 +623,96 @@ func (g *G) StartLoc() Location {
 	return Location{PC: g.StartPC, File: f, Line: l, Fn: fn}
 }
 
+var errTracebackAncestorsDisabled = errors.New("tracebackancestors is disabled")
+
+// Ancestors returns the list of ancestors for g.
+func (g *G) Ancestors(n int) ([]Ancestor, error) {
+	scope := globalScope(g.Thread.BinInfo(), g.Thread)
+	tbav, err := scope.EvalExpression("runtime.debug.tracebackancestors", loadSingleValue)
+	if err == nil && tbav.Unreadable == nil && tbav.Kind == reflect.Int {
+		tba, _ := constant.Int64Val(tbav.Value)
+		if tba == 0 {
+			return nil, errTracebackAncestorsDisabled
+		}
+	}
+
+	av, err := g.variable.structMember("ancestors")
+	if err != nil {
+		return nil, err
+	}
+	av = av.maybeDereference()
+	av.loadValue(LoadConfig{MaxArrayValues: n, MaxVariableRecurse: 1, MaxStructFields: -1})
+	if av.Unreadable != nil {
+		return nil, err
+	}
+	if av.Addr == 0 {
+		// no ancestors
+		return nil, nil
+	}
+
+	r := make([]Ancestor, len(av.Children))
+
+	for i := range av.Children {
+		if av.Children[i].Unreadable != nil {
+			r[i].Unreadable = av.Children[i].Unreadable
+			continue
+		}
+		goidv := av.Children[i].fieldVariable("goid")
+		if goidv.Unreadable != nil {
+			r[i].Unreadable = goidv.Unreadable
+			continue
+		}
+		r[i].ID, _ = constant.Int64Val(goidv.Value)
+		pcsVar := av.Children[i].fieldVariable("pcs")
+		if pcsVar.Unreadable != nil {
+			r[i].Unreadable = pcsVar.Unreadable
+		}
+		pcsVar.loaded = false
+		pcsVar.Children = pcsVar.Children[:0]
+		r[i].pcsVar = pcsVar
+	}
+
+	return r, nil
+}
+
+// Stack returns the stack trace of ancestor 'a' as saved by the runtime.
+func (a *Ancestor) Stack(n int) ([]Stackframe, error) {
+	if a.Unreadable != nil {
+		return nil, a.Unreadable
+	}
+	pcsVar := a.pcsVar.clone()
+	pcsVar.loadValue(LoadConfig{MaxArrayValues: n})
+	if pcsVar.Unreadable != nil {
+		return nil, pcsVar.Unreadable
+	}
+	r := make([]Stackframe, len(pcsVar.Children))
+	for i := range pcsVar.Children {
+		if pcsVar.Children[i].Unreadable != nil {
+			r[i] = Stackframe{Err: pcsVar.Children[i].Unreadable}
+			continue
+		}
+		if pcsVar.Children[i].Kind != reflect.Uint {
+			return nil, fmt.Errorf("wrong type for pcs item %d: %v", i, pcsVar.Children[i].Kind)
+		}
+		pc, _ := constant.Int64Val(pcsVar.Children[i].Value)
+		fn := a.pcsVar.bi.PCToFunc(uint64(pc))
+		if fn == nil {
+			loc := Location{PC: uint64(pc)}
+			r[i] = Stackframe{Current: loc, Call: loc}
+			continue
+		}
+		pc2 := uint64(pc)
+		if pc2-1 >= fn.Entry {
+			pc2--
+		}
+		f, ln := fn.cu.lineInfo.PCToLine(fn.Entry, pc2)
+		loc := Location{PC: uint64(pc), File: f, Line: ln, Fn: fn}
+		r[i] = Stackframe{Current: loc, Call: loc}
+	}
+	r[len(r)-1].Bottom = pcsVar.Len == int64(len(pcsVar.Children))
+	return r, nil
+}
+
 // Returns the list of saved return addresses used by stack barriers
 func (g *G) stkbar() ([]savedLR, error) {
 	if g.stkbarVar == nil { // stack barriers were removed in Go 1.9

pkg/terminal/command.go

@@ -251,11 +251,13 @@ When connected to a headless instance started with the --accept-multiclient, pas
 Show source around current point or provided linespec.`},
 		{aliases: []string{"stack", "bt"}, allowedPrefixes: onPrefix, cmdFn: stackCommand, helpMsg: `Print stack trace.
 
-	[goroutine <n>] [frame <m>] stack [<depth>] [-full] [-offsets] [-defer]
+	[goroutine <n>] [frame <m>] stack [<depth>] [-full] [-offsets] [-defer] [-a <n>] [-adepth <depth>]
 
 	-full		every stackframe is decorated with the value of its local variables and arguments.
 	-offsets	prints frame offset of each frame.
 	-defer		prints deferred function call stack for each frame.
+	-a <n>		prints stacktrace of n ancestors of the selected goroutine (target process must have tracebackancestors enabled)
+	-adepth <depth>	configures depth of ancestor stacktrace
 `},
 		{aliases: []string{"frame"},
 			cmdFn: func(t *Term, ctx callContext, arg string) error {
@@ -1412,6 +1414,20 @@ func stackCommand(t *Term, ctx callContext, args string) error {
 		return err
 	}
 	printStack(stack, "", sa.offsets)
+	if sa.ancestors > 0 {
+		ancestors, err := t.client.Ancestors(ctx.Scope.GoroutineID, sa.ancestors, sa.ancestorDepth)
+		if err != nil {
+			return err
+		}
+		for _, ancestor := range ancestors {
+			fmt.Printf("Created by Goroutine %d:\n", ancestor.ID)
+			if ancestor.Unreadable != "" {
+				fmt.Printf("\t%s\n", ancestor.Unreadable)
+				continue
+			}
+			printStack(ancestor.Stack, "\t", false)
+		}
+	}
 	return nil
 }
 
@@ -1420,6 +1436,9 @@ type stackArgs struct {
 	full       bool
 	offsets    bool
 	readDefers bool
+
+	ancestors     int
+	ancestorDepth int
 }
 
 func parseStackArgs(argstr string) (stackArgs, error) {
@@ -1429,7 +1448,18 @@ func parseStackArgs(argstr string) (stackArgs, error) {
 	}
 	if argstr != "" {
 		args := strings.Split(argstr, " ")
-		for i := range args {
+		for i := 0; i < len(args); i++ {
+			numarg := func(name string) (int, error) {
+				if i >= len(args) {
+					return 0, fmt.Errorf("expected number after %s", name)
+				}
+				n, err := strconv.Atoi(args[i])
+				if err != nil {
+					return 0, fmt.Errorf("expected number after %s: %v", name, err)
+				}
+				return n, nil
+
+			}
 			switch args[i] {
 			case "-full":
 				r.full = true
@@ -1437,6 +1467,20 @@ func parseStackArgs(argstr string) (stackArgs, error) {
 				r.offsets = true
 			case "-defer":
 				r.readDefers = true
+			case "-a":
+				i++
+				n, err := numarg("-a")
+				if err != nil {
+					return stackArgs{}, err
+				}
+				r.ancestors = n
+			case "-adepth":
+				i++
+				n, err := numarg("-adepth")
+				if err != nil {
+					return stackArgs{}, err
+				}
+				r.ancestorDepth = n
 			default:
 				n, err := strconv.Atoi(args[i])
 				if err != nil {
@@ -1446,6 +1490,9 @@ func parseStackArgs(argstr string) (stackArgs, error) {
 			}
 		}
 	}
+	if r.ancestors > 0 && r.ancestorDepth == 0 {
+		r.ancestorDepth = r.depth
+	}
 	return r, nil
 }
 

service/api/types.go

@@ -447,3 +447,11 @@ type Checkpoint struct {
 type Image struct {
 	Path string
 }
+
+// Ancestor represents a goroutine ancestor
+type Ancestor struct {
+	ID    int64
+	Stack []Stackframe
+
+	Unreadable string
+}

service/client.go

@@ -100,6 +100,9 @@ type Client interface {
 	// Returns stacktrace
 	Stacktrace(goroutineID int, depth int, readDefers bool, cfg *api.LoadConfig) ([]api.Stackframe, error)
 
+	// Returns ancestor stacktraces
+	Ancestors(goroutineID int, numAncestors int, depth int) ([]api.Ancestor, error)
+
 	// Returns whether we attached to a running process or not
 	AttachedToExistingProcess() bool
 

service/debugger/debugger.go

@@ -947,6 +947,48 @@ func (d *Debugger) Stacktrace(goroutineID, depth int, readDefers bool, cfg *proc
 	return d.convertStacktrace(rawlocs, cfg)
 }
 
+// Ancestors returns the stacktraces for the ancestors of a goroutine.
+func (d *Debugger) Ancestors(goroutineID, numAncestors, depth int) ([]api.Ancestor, error) {
+	d.processMutex.Lock()
+	defer d.processMutex.Unlock()
+
+	if _, err := d.target.Valid(); err != nil {
+		return nil, err
+	}
+
+	g, err := proc.FindGoroutine(d.target, goroutineID)
+	if err != nil {
+		return nil, err
+	}
+	if g == nil {
+		return nil, errors.New("no selected goroutine")
+	}
+
+	ancestors, err := g.Ancestors(numAncestors)
+	if err != nil {
+		return nil, err
+	}
+
+	r := make([]api.Ancestor, len(ancestors))
+	for i := range ancestors {
+		r[i].ID = ancestors[i].ID
+		if ancestors[i].Unreadable != nil {
+			r[i].Unreadable = ancestors[i].Unreadable.Error()
+			continue
+		}
+		frames, err := ancestors[i].Stack(depth)
+		if err != nil {
+			r[i].Unreadable = fmt.Sprintf("could not read ancestor stacktrace: %v", err)
+			continue
+		}
+		r[i].Stack, err = d.convertStacktrace(frames, nil)
+		if err != nil {
+			r[i].Unreadable = fmt.Sprintf("could not read ancestor stacktrace: %v", err)
+		}
+	}
+	return r, nil
+}
+
 func (d *Debugger) convertStacktrace(rawlocs []proc.Stackframe, cfg *proc.LoadConfig) ([]api.Stackframe, error) {
 	locations := make([]api.Stackframe, 0, len(rawlocs))
 	for i := range rawlocs {

service/rpc2/client.go

@@ -310,6 +310,12 @@ func (c *RPCClient) Stacktrace(goroutineId, depth int, readDefers bool, cfg *api
 	return out.Locations, err
 }
 
+func (c *RPCClient) Ancestors(goroutineID int, numAncestors int, depth int) ([]api.Ancestor, error) {
+	var out AncestorsOut
+	err := c.call("Ancestors", AncestorsIn{goroutineID, numAncestors, depth}, &out)
+	return out.Ancestors, err
+}
+
 func (c *RPCClient) AttachedToExistingProcess() bool {
 	out := new(AttachedToExistingProcessOut)
 	c.call("AttachedToExistingProcess", AttachedToExistingProcessIn{}, out)

service/rpc2/server.go

@@ -174,10 +174,24 @@ func (s *RPCServer) Stacktrace(arg StacktraceIn, out *StacktraceOut) error {
 	}
 	var err error
 	out.Locations, err = s.debugger.Stacktrace(arg.Id, arg.Depth, arg.Defers, api.LoadConfigToProc(cfg))
-	if err != nil {
-		return err
-	}
-	return nil
+	return err
+}
+
+type AncestorsIn struct {
+	GoroutineID  int
+	NumAncestors int
+	Depth        int
+}
+
+type AncestorsOut struct {
+	Ancestors []api.Ancestor
+}
+
+// Ancestors returns the stacktraces for the ancestors of a goroutine.
+func (s *RPCServer) Ancestors(arg AncestorsIn, out *AncestorsOut) error {
+	var err error
+	out.Ancestors, err = s.debugger.Ancestors(arg.GoroutineID, arg.NumAncestors, arg.Depth)
+	return err
 }
 
 type ListBreakpointsIn struct {

service/test/integration2_test.go

@@ -1640,3 +1640,36 @@ func TestClientServerFunctionCallStacktrace(t *testing.T) {
 		}
 	})
 }
+
+func TestAncestors(t *testing.T) {
+	if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 11) {
+		t.Skip("not supported on Go <= 1.10")
+	}
+	savedGodebug := os.Getenv("GODEBUG")
+	os.Setenv("GODEBUG", "tracebackancestors=100")
+	defer os.Setenv("GODEBUG", savedGodebug)
+	withTestClient2("testnextprog", t, func(c service.Client) {
+		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.testgoroutine", Line: -1})
+		assertNoError(err, t, "CreateBreakpoin")
+		state := <-c.Continue()
+		assertNoError(state.Err, t, "Continue()")
+		ancestors, err := c.Ancestors(-1, 1000, 1000)
+		assertNoError(err, t, "Ancestors")
+		t.Logf("ancestors: %#v\n", ancestors)
+		if len(ancestors) != 1 {
+			t.Fatalf("expected only one ancestor got %d", len(ancestors))
+		}
+
+		mainFound := false
+		for _, ancestor := range ancestors {
+			for _, frame := range ancestor.Stack {
+				if frame.Function.Name() == "main.main" {
+					mainFound = true
+				}
+			}
+		}
+		if !mainFound {
+			t.Fatal("function main.main not found in any ancestor")
+		}
+	})
+}