pkg/terminal,service/debugger: Support to add a new suboption --follow-calls to trace subcommand (#3594)

* rebasing on master to implement --followcalls

* in progress changes to enable --followcalls

* rebase to master: modified function to add children to funcs array

* modify main traversal loop

* added tests to check different scenarios

* added tests to check different scenarios

* added tests to check different scenarios

* add test to check for overlapping regular expression

* modified type of strings array as a return only

* changed depth to a simple integer instead of a global map

* avoid calling traverse on recursive calls

* Added tests for various call graphs to test trace followfuncs

* Added tests for various call graphs to test trace followfuncs

* Added tests for various call graphs to test trace followfuncs

* made auxillary changes for build to go through for new option follow-calls

* Add support to print depth of the function calls as well

* Added two sample output files for checking

* Bypass morestack_noctxt in output for verification testing

* Corrected newline error by adding newlines only if the line does not match morestack_noctxt

* Added more tests

* Cleanup

* Updated documentation

* fixed error message in fmt.Errorf

* Fixed result of Errorf not used error

* Addressing review comments to fix depth reporting and other issues

* dont invoke stacktrace if tracefollowcalls is enabled, compute depth from main regex root symbol than main.main

* Addressing a part of review comments

* Added changes to allow deferred functions to be picked up for tracing

* Fix issue to avoid printing stack for a simple trace option

* Moving most tests to integration2_test.go and keeping only one in dlv_test.go

* Moving most tests to integration2_test.go and keeping only one in dlv_test.go

* Adding panic-defer test case

* Moved rest of the tests to integration2_test.go

* addressing review comments: folding Functions and FunctionsDeep, reducing branches by using depth prefix, wrap using %w and other comments

* Optimize traversal and parts of printing trace point function and modify trace output layout
and adjust tests accordingly

* Resolved error occurring due to staticcheck

* Implemented traversal algorithm using breadth first search

* Addressing review comments on the breadth first search implementation and
other comments

* Inline filterRuntimeFuncs and remove duplicate initialization

Documentation/cli/starlark.md

@@ -50,7 +50,7 @@ breakpoints(All) | Equivalent to API call [ListBreakpoints](https://godoc.org/gi
 checkpoints() | Equivalent to API call [ListCheckpoints](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListCheckpoints)
 dynamic_libraries() | Equivalent to API call [ListDynamicLibraries](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListDynamicLibraries)
 function_args(Scope, Cfg) | Equivalent to API call [ListFunctionArgs](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctionArgs)
-functions(Filter) | Equivalent to API call [ListFunctions](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctions)
+functions(Filter, FollowCalls) | Equivalent to API call [ListFunctions](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctions)
 goroutines(Start, Count, Filters, GoroutineGroupingOptions, EvalScope) | Equivalent to API call [ListGoroutines](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListGoroutines)
 local_vars(Scope, Cfg) | Equivalent to API call [ListLocalVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListLocalVars)
 package_vars(Filter, Cfg) | Equivalent to API call [ListPackageVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListPackageVars)

Documentation/usage/dlv_trace.md

@@ -23,6 +23,7 @@ dlv trace [package] regexp [flags]
 ```
       --ebpf               Trace using eBPF (experimental).
   -e, --exec string        Binary file to exec and trace.
+      --follow-calls int   Trace all children of the function to the required depth
   -h, --help               help for trace
       --output string      Output path for the binary.
   -p, --pid int            Pid to attach to.

_fixtures/leaf4.go

@@ -0,0 +1,23 @@
+package main
+
+import "fmt"
+
+func D(i int) int {
+	return i * i * i
+}
+func C(i int) int {
+
+	return i + 20
+}
+func B(i int) int {
+	d := C(i) + 40
+	return d + D(i)
+}
+func A(i int) int {
+	return 10 + B(i)
+}
+func main() {
+	j := 0
+	j += A(2)
+	fmt.Println(j)
+}

_fixtures/leafcommon.go

@@ -0,0 +1,23 @@
+package main
+
+import "fmt"
+
+func D(i int) int {
+	return i * i * i
+}
+func C(i int) int {
+
+	return D(i+10) + 20
+}
+func B(i int) int {
+	return i * D(i)
+}
+func A(i int) int {
+	d := 10 + B(i)
+	return d + C(i)
+}
+func main() {
+	j := 0
+	j += A(2)
+	fmt.Println(j)
+}

_fixtures/leafindrec.go

@@ -0,0 +1,23 @@
+package main
+
+import "fmt"
+
+func B(i int) int {
+	if i > 0 {
+		return A(i - 1)
+	} else {
+		return 0
+	}
+}
+func A(n int) int {
+	if n <= 1 {
+		return n
+	} else {
+		return B(n - 3)
+	}
+}
+func main() {
+	j := 0
+	j += B(12)
+	fmt.Println(j)
+}

_fixtures/leafrec.go

@@ -0,0 +1,17 @@
+package main
+
+import "fmt"
+
+func A(i int, n int) int {
+	if n == 1 {
+		return i
+	} else {
+		n--
+		return (i * A(i-1, n))
+	}
+}
+func main() {
+	j := 0
+	j += A(5, 5)
+	fmt.Println(j)
+}

_fixtures/leafregex.go

@@ -0,0 +1,23 @@
+package main
+
+import "fmt"
+
+func callmed(i int) int {
+	return i * i * i
+}
+func callmee(i int) int {
+
+	return i + 20
+}
+func callme2(i int) int {
+	d := callmee(i) + 40
+	return d + callmed(i)
+}
+func callme(i int) int {
+	return 10 + callme2(i)
+}
+func main() {
+	j := 0
+	j += callme(2)
+	fmt.Println(j)
+}

_fixtures/panicex.go

@@ -0,0 +1,28 @@
+package main
+
+        func F0() {
+                defer func() {
+                        recover()
+                }()
+                F1()
+        }
+
+        func F1() {
+                F2()
+        }
+
+        func F2() {
+                F3()
+        }
+
+        func F3() {
+                F4()
+        }
+
+        func F4() {
+                panic("blah")
+        }
+
+        func main() {
+                F0()
+        }

_fixtures/testtracefns.go

@@ -0,0 +1,81 @@
+package main
+
+import "fmt"
+
+func D(i int) int {
+	return i * i * i
+}
+func C(i int) int {
+
+	return D(i+10) + 20
+}
+func B(i int) int {
+	return i * D(i)
+}
+func A(i int) int {
+	d := 10 + B(i)
+	return d + C(i)
+}
+func second(i int) int {
+	if i > 0 {
+		return first(i - 1)
+	} else {
+		return 0
+	}
+}
+func first(n int) int {
+	if n <= 1 {
+		return n
+	} else {
+		return second(n - 3)
+	}
+}
+
+func callmed(i int) int {
+	return i * i * i
+}
+func callmee(i int) int {
+
+	return i + 20
+}
+func callme2(i int) int {
+	d := callmee(i) + 40
+	return d + callmed(i)
+}
+func callme(i int) int {
+	return 10 + callme2(i)
+}
+
+func F0() {
+	defer func() {
+		recover()
+	}()
+	F1()
+}
+
+func F1() {
+	F2()
+}
+
+func F2() {
+	F3()
+}
+
+func F3() {
+	F4()
+}
+
+func F4() {
+	panic("blah")
+}
+
+func main() {
+	j := 0
+	j += A(2)
+
+	j += first(6)
+	j += callme(2)
+	fmt.Println(j)
+	F0()
+
+}

cmd/dlv/cmds/commands.go

@@ -88,6 +88,7 @@ var (
 	traceStackDepth    int
 	traceUseEBPF       bool
 	traceShowTimestamp bool
+	traceFollowCalls   int
 
 	// redirect specifications for target process
 	redirects []string
@@ -363,6 +364,7 @@ only see the output of the trace operations you can redirect stdout.`,
 	must(traceCommand.RegisterFlagCompletionFunc("stack", cobra.NoFileCompletions))
 	traceCommand.Flags().String("output", "", "Output path for the binary.")
 	must(traceCommand.MarkFlagFilename("output"))
+	traceCommand.Flags().IntVarP(&traceFollowCalls, "follow-calls", "", 0, "Trace all children of the function to the required depth")
 	rootCommand.AddCommand(traceCommand)
 
 	coreCommand := &cobra.Command{
@@ -702,6 +704,10 @@ func traceCmd(cmd *cobra.Command, args []string, conf *config.Config) int {
 
 			processArgs = append([]string{debugname}, targetArgs...)
 		}
+		if dlvArgsLen >= 3 && traceFollowCalls <= 0 {
+			fmt.Fprintln(os.Stderr, "Need to specify a trace depth of atleast 1")
+			return 1
+		}
 
 		// Make a local in-memory connection that client and server use to communicate
 		listener, clientConn := service.ListenerPipe()
@@ -738,8 +744,7 @@ func traceCmd(cmd *cobra.Command, args []string, conf *config.Config) int {
 			<-ch
 			client.Halt()
 		}()
-
-		funcs, err := client.ListFunctions(regexp)
+		funcs, err := client.ListFunctions(regexp, traceFollowCalls)
 		if err != nil {
 			fmt.Fprintln(os.Stderr, err)
 			return 1
@@ -755,13 +760,22 @@ func traceCmd(cmd *cobra.Command, args []string, conf *config.Config) int {
 				}
 			} else {
 				// Fall back to breakpoint based tracing if we get an error.
+				var stackdepth int
+				// Default size of stackdepth to trace function calls and descendants=20
+				stackdepth = traceStackDepth
+				if traceFollowCalls > 0 && stackdepth == 0 {
+					stackdepth = 20
+				}
 				_, err = client.CreateBreakpoint(&api.Breakpoint{
 					FunctionName:     funcs[i],
 					Tracepoint:       true,
 					Line:             -1,
-					Stacktrace:   traceStackDepth,
+					Stacktrace:       stackdepth,
 					LoadArgs:         &terminal.ShortLoadConfig,
+					TraceFollowCalls: traceFollowCalls,
+					RootFuncName:     regexp,
 				})
+
 				if err != nil && !isBreakpointExistsErr(err) {
 					fmt.Fprintf(os.Stderr, "unable to set tracepoint on function %s: %#v\n", funcs[i], err)
 					continue
@@ -777,9 +791,11 @@ func traceCmd(cmd *cobra.Command, args []string, conf *config.Config) int {
 					_, err = client.CreateBreakpoint(&api.Breakpoint{
 						Addr:             addrs[i],
 						TraceReturn:      true,
-						Stacktrace:  traceStackDepth,
+						Stacktrace:       stackdepth,
 						Line:             -1,
 						LoadArgs:         &terminal.ShortLoadConfig,
+						TraceFollowCalls: traceFollowCalls,
+						RootFuncName:     regexp,
 					})
 					if err != nil && !isBreakpointExistsErr(err) {
 						fmt.Fprintf(os.Stderr, "unable to set tracepoint on function %s: %#v\n", funcs[i], err)

cmd/dlv/dlv_test.go

@@ -970,6 +970,39 @@ func TestTrace2(t *testing.T) {
 	assertNoError(cmd.Wait(), t, "cmd.Wait()")
 }
 
+func TestTraceDirRecursion(t *testing.T) {
+	dlvbin := getDlvBin(t)
+
+	expected := []byte("> goroutine(1):frame(1) main.A(5, 5)\n > goroutine(1):frame(2) main.A(4, 4)\n  > goroutine(1):frame(3) main.A(3, 3)\n   > goroutine(1):frame(4) main.A(2, 2)\n    > goroutine(1):frame(5) main.A(1, 1)\n    >> goroutine(1):frame(5) main.A => (1)\n   >> goroutine(1):frame(4) main.A => (2)\n  >> goroutine(1):frame(3) main.A => (6)\n >> goroutine(1):frame(2) main.A => (24)\n>> goroutine(1):frame(1) main.A => (120)\n")
+
+	fixtures := protest.FindFixturesDir()
+	cmd := exec.Command(dlvbin, "trace", "--output", filepath.Join(t.TempDir(), "__debug"), filepath.Join(fixtures, "leafrec.go"), "main.A", "--follow-calls", "4")
+	rdr, err := cmd.StderrPipe()
+	assertNoError(err, t, "stderr pipe")
+	defer rdr.Close()
+
+	cmd.Dir = filepath.Join(fixtures, "buildtest")
+
+	assertNoError(cmd.Start(), t, "running trace")
+	// Parse output to ignore calls to morestack_noctxt for comparison
+	scan := bufio.NewScanner(rdr)
+	text := ""
+	outputtext := ""
+	for scan.Scan() {
+		text = scan.Text()
+		if !strings.Contains(text, "morestack_noctxt") {
+			outputtext += text
+			outputtext += "\n"
+		}
+	}
+	output := []byte(outputtext)
+
+	if !bytes.Contains(output, expected) {
+		t.Fatalf("expected:\n%s\ngot:\n%s", string(expected), string(output))
+	}
+	assertNoError(cmd.Wait(), t, "cmd.Wait()")
+}
+
 func TestTraceMultipleGoroutines(t *testing.T) {
 	dlvbin := getDlvBin(t)
 

pkg/proc/breakpoints.go

@@ -63,6 +63,11 @@ type Breakpoint struct {
 	// ReturnInfo describes how to collect return variables when this
 	// breakpoint is hit as a return breakpoint.
 	returnInfo *returnBreakpointInfo
+
+	// RootFuncName is the name of the root function from where tracing needs to be done
+	RootFuncName string
+	// TraceFollowCalls indicates the depth of tracing
+	TraceFollowCalls int
 }
 
 // Breaklet represents one of multiple breakpoints that can overlap on a
@@ -1016,6 +1021,10 @@ type LogicalBreakpoint struct {
 	Cond ast.Expr
 
 	UserData interface{} // Any additional information about the breakpoint
+	// Name of root function from where tracing needs to be done
+	RootFuncName string
+	// depth of tracing
+	TraceFollowCalls int
 }
 
 // SetBreakpoint describes how a breakpoint should be set.

pkg/terminal/command.go

@@ -2331,7 +2331,7 @@ func packages(t *Term, ctx callContext, args string) error {
 }
 
 func funcs(t *Term, ctx callContext, args string) error {
-	return t.printSortedStrings(t.client.ListFunctions(args))
+	return t.printSortedStrings(t.client.ListFunctions(args, 0))
 }
 
 func types(t *Term, ctx callContext, args string) error {
@@ -2908,21 +2908,54 @@ func printBreakpointInfo(t *Term, th *api.Thread, tracepointOnNewline bool) {
 			fmt.Fprintf(t.stdout, "\t%s: %s\n", v.Name, v.MultilineString("\t", ""))
 		}
 	}
-
 	if bpi.Stacktrace != nil {
+		// TraceFollowCalls and Stacktrace are mutually exclusive as they pollute each others outputs
+		if th.Breakpoint.TraceFollowCalls <= 0 {
 			tracepointnl()
 			fmt.Fprintf(t.stdout, "\tStack:\n")
 			printStack(t, t.stdout, bpi.Stacktrace, "\t\t", false)
 		}
 	}
+}
 
 func printTracepoint(t *Term, th *api.Thread, bpname string, fn *api.Function, args string, hasReturnValue bool) {
 	if t.conf.TraceShowTimestamp {
 		fmt.Fprintf(t.stdout, "%s ", time.Now().Format(time.RFC3339Nano))
 	}
 
+	var sdepth, rootindex int
+	depthPrefix := ""
+	tracePrefix := ""
+	if th.Breakpoint.TraceFollowCalls > 0 {
+		// Trace Follow Calls; stack is required to calculate depth of functions
+		rootindex = -1
+		if th.BreakpointInfo == nil || th.BreakpointInfo.Stacktrace == nil {
+			return
+		}
+
+		stack := th.BreakpointInfo.Stacktrace
+		for i := len(stack) - 1; i >= 0; i-- {
+			if stack[i].Function.Name() == th.Breakpoint.RootFuncName {
+				if rootindex == -1 {
+					rootindex = i
+					break
+				}
+			}
+		}
+		sdepth = rootindex + 1
+		tracePrefix = fmt.Sprintf("goroutine(%d):frame(%d)", th.GoroutineID, sdepth)
+		if sdepth > 0 {
+			depthPrefix = strings.Repeat(" ", sdepth-1)
+		}
+	} else {
+		tracePrefix = fmt.Sprintf("goroutine(%d):", th.GoroutineID)
+	}
+
 	if th.Breakpoint.Tracepoint {
-		fmt.Fprintf(t.stdout, "> goroutine(%d): %s%s(%s)\n", th.GoroutineID, bpname, fn.Name(), args)
+		// Print trace only if there was a match on the function while TraceFollowCalls is on or if it's a regular trace
+		if rootindex != -1 || th.Breakpoint.TraceFollowCalls <= 0 {
+			fmt.Fprintf(t.stdout, "%s> %s %s%s(%s)\n", depthPrefix, tracePrefix, bpname, fn.Name(), args)
+		}
 		printBreakpointInfo(t, th, !hasReturnValue)
 	}
 	if th.Breakpoint.TraceReturn {
@@ -2930,7 +2963,14 @@ func printTracepoint(t *Term, th *api.Thread, bpname string, fn *api.Function, a
 		for _, v := range th.ReturnValues {
 			retVals = append(retVals, v.SinglelineString())
 		}
-		fmt.Fprintf(t.stdout, ">> goroutine(%d): %s => (%s)\n", th.GoroutineID, fn.Name(), strings.Join(retVals, ","))
+		// Print trace only if there was a match on the function while TraceFollowCalls is on or if it's a regular trace
+		if rootindex != -1 || th.Breakpoint.TraceFollowCalls <= 0 {
+			fmt.Fprintf(t.stdout, "%s>> %s %s => (%s)\n", depthPrefix, tracePrefix, fn.Name(), strings.Join(retVals, ","))
+		}
+	}
+	if th.Breakpoint.TraceFollowCalls > 0 {
+		// As of now traceFollowCalls and Stacktrace are mutually exclusive options
+		return
 	}
 	if th.Breakpoint.TraceReturn || !hasReturnValue {
 		if th.BreakpointInfo != nil && th.BreakpointInfo.Stacktrace != nil {
@@ -3520,7 +3560,7 @@ func (t *Term) formatBreakpointLocation(bp *api.Breakpoint) string {
 }
 
 func shouldAskToSuspendBreakpoint(t *Term) bool {
-	fns, _ := t.client.ListFunctions(`^plugin\.Open$`)
+	fns, _ := t.client.ListFunctions(`^plugin\.Open$`, 0)
 	_, err := t.client.GetState()
 	return len(fns) > 0 || isErrProcessExited(err) || t.client.FollowExecEnabled()
 }

pkg/terminal/starbind/starlark_mapping.go

@@ -1094,11 +1094,19 @@ func (env *Env) starlarkPredeclare() (starlark.StringDict, map[string]string) {
 				return starlark.None, decorateError(thread, err)
 			}
 		}
+		if len(args) > 1 && args[1] != starlark.None {
+			err := unmarshalStarlarkValue(args[1], &rpcArgs.FollowCalls, "FollowCalls")
+			if err != nil {
+				return starlark.None, decorateError(thread, err)
+			}
+		}
 		for _, kv := range kwargs {
 			var err error
 			switch kv[0].(starlark.String) {
 			case "Filter":
 				err = unmarshalStarlarkValue(kv[1], &rpcArgs.Filter, "Filter")
+			case "FollowCalls":
+				err = unmarshalStarlarkValue(kv[1], &rpcArgs.FollowCalls, "FollowCalls")
 			default:
 				err = fmt.Errorf("unknown argument %q", kv[0])
 			}
@@ -1112,7 +1120,7 @@ func (env *Env) starlarkPredeclare() (starlark.StringDict, map[string]string) {
 		}
 		return env.interfaceToStarlarkValue(rpcRet), nil
 	})
-	doc["functions"] = "builtin functions(Filter)\n\nfunctions lists all functions in the process matching filter."
+	doc["functions"] = "builtin functions(Filter, FollowCalls)\n\nfunctions lists all functions in the process matching filter."
 	r["goroutines"] = starlark.NewBuiltin("goroutines", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
 		if err := isCancelled(thread); err != nil {
 			return starlark.None, decorateError(thread, err)

pkg/terminal/terminal.go

@@ -265,7 +265,7 @@ func (t *Term) Run() (int, error) {
 
 	fns := trie.New()
 	cmds := trie.New()
-	funcs, _ := t.client.ListFunctions("")
+	funcs, _ := t.client.ListFunctions("", 0)
 	for _, fn := range funcs {
 		fns.Add(fn, nil)
 	}

service/api/conversions.go

@@ -34,6 +34,8 @@ func ConvertLogicalBreakpoint(lbp *proc.LogicalBreakpoint) *Breakpoint {
 		TotalHitCount:    lbp.TotalHitCount,
 		Disabled:         !lbp.Enabled,
 		UserData:         lbp.UserData,
+		RootFuncName:     lbp.RootFuncName,
+		TraceFollowCalls: lbp.TraceFollowCalls,
 	}
 
 	b.HitCount = map[string]uint64{}

service/api/types.go

@@ -133,6 +133,11 @@ type Breakpoint struct {
 	Disabled bool `json:"disabled"`
 
 	UserData interface{} `json:"-"`
+
+	// RootFuncName is the Root function from where tracing needs to be done
+	RootFuncName string
+	// TraceFollowCalls indicates the Depth of tracing
+	TraceFollowCalls int
 }
 
 // ValidBreakpointName returns an error if

service/client.go

@@ -105,7 +105,7 @@ type Client interface {
 	// ListSources lists all source files in the process matching filter.
 	ListSources(filter string) ([]string, error)
 	// ListFunctions lists all functions in the process matching filter.
-	ListFunctions(filter string) ([]string, error)
+	ListFunctions(filter string, tracefollow int) ([]string, error)
 	// ListTypes lists all types in the process matching filter.
 	ListTypes(filter string) ([]string, error)
 	// ListPackagesBuildInfo lists all packages in the process matching filter.

service/debugger/debugger.go

@@ -494,6 +494,7 @@ func (d *Debugger) Restart(rerecord bool, pos string, resetArgs bool, newArgs []
 
 	if !resetArgs && (d.config.Stdout.File != nil || d.config.Stderr.File != nil) {
 		return nil, ErrCanNotRestart
+
 	}
 
 	if err := d.detach(true); err != nil {
@@ -915,6 +916,8 @@ func copyLogicalBreakpointInfo(lbp *proc.LogicalBreakpoint, requested *api.Break
 	lbp.LoadArgs = api.LoadConfigToProc(requested.LoadArgs)
 	lbp.LoadLocals = api.LoadConfigToProc(requested.LoadLocals)
 	lbp.UserData = requested.UserData
+	lbp.RootFuncName = requested.RootFuncName
+	lbp.TraceFollowCalls = requested.TraceFollowCalls
 	lbp.Cond = nil
 	if requested.Cond != "" {
 		var err error
@@ -1452,7 +1455,7 @@ func uniq(s []string) []string {
 }
 
 // Functions returns a list of functions in the target process.
-func (d *Debugger) Functions(filter string) ([]string, error) {
+func (d *Debugger) Functions(filter string, followCalls int) ([]string, error) {
 	d.targetMutex.Lock()
 	defer d.targetMutex.Unlock()
 
@@ -1466,15 +1469,85 @@ func (d *Debugger) Functions(filter string) ([]string, error) {
 	for t.Next() {
 		for _, f := range t.BinInfo().Functions {
 			if regex.MatchString(f.Name) {
+				if followCalls > 0 {
+					newfuncs, err := traverse(t, &f, 1, followCalls)
+					if err != nil {
+						return nil, fmt.Errorf("traverse failed with error %w", err)
+					}
+					funcs = append(funcs, newfuncs...)
+				} else {
 					funcs = append(funcs, f.Name)
 				}
 			}
 		}
+	}
 	sort.Strings(funcs)
 	funcs = uniq(funcs)
 	return funcs, nil
 }
 
+func traverse(t proc.ValidTargets, f *proc.Function, depth int, followCalls int) ([]string, error) {
+	type TraceFunc struct {
+		Func    *proc.Function
+		Depth   int
+		visited bool
+	}
+	type TraceFuncptr *TraceFunc
+
+	TraceMap := make(map[string]TraceFuncptr)
+	queue := make([]TraceFuncptr, 0, 40)
+	funcs := []string{}
+	rootnode := &TraceFunc{Func: new(proc.Function), Depth: depth, visited: false}
+	rootnode.Func = f
+
+	// cache function details in a map for reuse
+	TraceMap[f.Name] = rootnode
+	queue = append(queue, rootnode)
+	for len(queue) > 0 {
+		parent := queue[0]
+		queue = queue[1:]
+		if parent == nil {
+			panic("attempting to open file Delve cannot parse")
+		}
+		if parent.Depth > followCalls {
+			continue
+		}
+		if !parent.visited {
+			funcs = append(funcs, parent.Func.Name)
+			parent.visited = true
+		} else if parent.visited {
+			continue
+		}
+
+		if parent.Depth+1 > followCalls {
+			// Avoid diassembling if we already cross the follow-calls depth
+			continue
+		}
+		f := parent.Func
+		text, err := proc.Disassemble(t.Memory(), nil, t.Breakpoints(), t.BinInfo(), f.Entry, f.End)
+		if err != nil {
+			return nil, fmt.Errorf("disassemble failed with error %w", err)
+		}
+		for _, instr := range text {
+			if instr.IsCall() && instr.DestLoc != nil && instr.DestLoc.Fn != nil {
+				cf := instr.DestLoc.Fn
+				if ((strings.HasPrefix(cf.Name, "runtime.") || strings.HasPrefix(cf.Name, "runtime/internal")) && cf.Name != "runtime.deferreturn" && cf.Name != "runtime.gorecover" && cf.Name != "runtime.gopanic") {
+					continue
+				}
+				childnode := TraceMap[cf.Name]
+				if childnode == nil {
+					childnode = &TraceFunc{Func: nil, Depth: parent.Depth + 1, visited: false}
+					childnode.Func = cf
+					TraceMap[cf.Name] = childnode
+					queue = append(queue, childnode)
+				}
+
+			}
+		}
+	}
+	return funcs, nil
+}
+
 // Types returns all type information in the binary.
 func (d *Debugger) Types(filter string) ([]string, error) {
 	d.targetMutex.Lock()
@@ -1922,6 +1995,7 @@ func (d *Debugger) convertDefers(defers []*proc.Defer) []api.Defer {
 				SP: defers[i].SP,
 			}
 		}
+
 	}
 
 	return r

service/rpc1/server.go

@@ -263,8 +263,8 @@ func (s *RPCServer) ListSources(filter string, sources *[]string) error {
 	return nil
 }
 
-func (s *RPCServer) ListFunctions(filter string, funcs *[]string) error {
-	fns, err := s.debugger.Functions(filter)
+func (s *RPCServer) ListFunctions(filter string, followCalls int, funcs *[]string) error {
+	fns, err := s.debugger.Functions(filter, followCalls)
 	if err != nil {
 		return err
 	}

service/rpc2/client.go

@@ -347,9 +347,9 @@ func (c *RPCClient) ListSources(filter string) ([]string, error) {
 	return sources.Sources, err
 }
 
-func (c *RPCClient) ListFunctions(filter string) ([]string, error) {
+func (c *RPCClient) ListFunctions(filter string, TraceFollow int) ([]string, error) {
 	funcs := new(ListFunctionsOut)
-	err := c.call("ListFunctions", ListFunctionsIn{filter}, funcs)
+	err := c.call("ListFunctions", ListFunctionsIn{filter, TraceFollow}, funcs)
 	return funcs.Funcs, err
 }
 

service/rpc2/server.go

@@ -579,6 +579,7 @@ func (s *RPCServer) ListSources(arg ListSourcesIn, out *ListSourcesOut) error {
 
 type ListFunctionsIn struct {
 	Filter      string
+	FollowCalls int
 }
 
 type ListFunctionsOut struct {
@@ -587,7 +588,7 @@ type ListFunctionsOut struct {
 
 // ListFunctions lists all functions in the process matching filter.
 func (s *RPCServer) ListFunctions(arg ListFunctionsIn, out *ListFunctionsOut) error {
-	fns, err := s.debugger.Functions(arg.Filter)
+	fns, err := s.debugger.Functions(arg.Filter, arg.FollowCalls)
 	if err != nil {
 		return err
 	}

service/test/integration2_test.go

@@ -798,6 +798,42 @@ func TestClientServer_infoLocals(t *testing.T) {
 	})
 }
 
+func matchFunctions(t *testing.T, funcs []string, expected []string, depth int) {
+	for i := range funcs {
+		if funcs[i] != expected[i] {
+			t.Fatalf("Function %s  not found in ListFunctions --follow-calls=%d output", expected[i], depth)
+		}
+	}
+}
+
+func TestTraceFollowCallsCommand(t *testing.T) {
+	protest.AllowRecording(t)
+	withTestClient2("testtracefns", t, func(c service.Client) {
+		depth := 3
+		functions, err := c.ListFunctions("main.A", depth)
+		assertNoError(err, t, "ListFunctions()")
+		expected := []string{"main.A", "main.B", "main.C", "main.D"}
+		matchFunctions(t, functions, expected, depth)
+
+		functions, err = c.ListFunctions("main.first", depth)
+		assertNoError(err, t, "ListFunctions()")
+		expected = []string{"main.first", "main.second"}
+		matchFunctions(t, functions, expected, depth)
+
+		depth = 4
+		functions, err = c.ListFunctions("main.callme", depth)
+		assertNoError(err, t, "ListFunctions()")
+		expected = []string{"main.callme", "main.callme2", "main.callmed", "main.callmee"}
+		matchFunctions(t, functions, expected, depth)
+
+		depth = 6
+		functions, err = c.ListFunctions("main.F0", depth)
+		assertNoError(err, t, "ListFunctions()")
+		expected = []string{"main.F0", "main.F0.func1", "main.F1", "main.F2", "main.F3", "main.F4", "runtime.deferreturn", "runtime.gopanic", "runtime.gorecover"}
+		matchFunctions(t, functions, expected, depth)
+	})
+}
+
 func TestClientServer_infoArgs(t *testing.T) {
 	protest.AllowRecording(t)
 	withTestClient2("testnextprog", t, func(c service.Client) {