terminal,service: Add filtering and grouping to goroutines command (#2504)

Adds filtering and grouping to the goroutines command.

The current implementation of the goroutines command is modeled after
the threads command of gdb. It works well for programs that have up to
a couple dozen goroutines but becomes unusable quickly after that.

This commit adds the ability to filter and group goroutines by several
different properties, allowing a better debugging experience on
programs that have hundreds or thousands of goroutines.

Documentation/cli/README.md

@@ -353,18 +353,62 @@ Aliases: gr
 ## goroutines
 List program goroutines.
 
-	goroutines [-u (default: user location)|-r (runtime location)|-g (go statement location)|-s (start location)] [-t (stack trace)] [-l (labels)]
+	goroutines [-u|-r|-g|-s] [-t [depth]] [-l] [-with loc expr] [-without loc expr] [-group argument]
 
 Print out info for every goroutine. The flag controls what information is shown along with each goroutine:
 
-	-u	displays location of topmost stackframe in user code
+	-u	displays location of topmost stackframe in user code (default)
 	-r	displays location of topmost stackframe (including frames inside private runtime functions)
 	-g	displays location of go instruction that created the goroutine
 	-s	displays location of the start function
-	-t	displays goroutine's stacktrace
+	-t	displays goroutine's stacktrace (an optional depth value can be specified, default: 10)
 	-l	displays goroutine's labels
 
-If no flag is specified the default is -u.
+If no flag is specified the default is -u, i.e. the first frame within the first 30 frames that is not executing a runtime private function.
+
+FILTERING
+
+If -with or -without are specified only goroutines that match the given condition are returned.
+
+To only display goroutines where the specified location contains (or does not contain, for -without and -wo) expr as a substring, use:
+
+	goroutines -with (userloc|curloc|goloc|startloc) expr
+	goroutines -w (userloc|curloc|goloc|startloc) expr
+	goroutines -without (userloc|curloc|goloc|startloc) expr
+	goroutines -wo (userloc|curloc|goloc|startloc) expr
+	
+To only display goroutines that have (or do not have) the specified label key and value, use:
+	
+
+	goroutines -with label key=value
+	goroutines -without label key=value
+	
+To only display goroutines that have (or do not have) the specified label key, use:
+
+	goroutines -with label key
+	goroutines -without label key
+	
+To only display goroutines that are running (or are not running) on a OS thread, use:
+
+
+	goroutines -with running
+	goroutines -without running
+	
+To only display user (or runtime) goroutines, use:
+
+	goroutines -with user
+	goroutines -without user
+
+GROUPING
+
+	goroutines -group (userloc|curloc|goloc|startloc|running|user)
+
+Groups goroutines by the given location, running status or user classification, up to 5 goroutines per group will be displayed as well as the total number of goroutines in the group.
+
+	goroutines -group label key
+
+Groups goroutines by the value of the label with the specified key.
+
 
 Aliases: grs
 

Documentation/cli/starlark.md

@@ -45,7 +45,7 @@ checkpoints() | Equivalent to API call [ListCheckpoints](https://godoc.org/githu
 dynamic_libraries() | Equivalent to API call [ListDynamicLibraries](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListDynamicLibraries)
 function_args(Scope, Cfg) | Equivalent to API call [ListFunctionArgs](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctionArgs)
 functions(Filter) | Equivalent to API call [ListFunctions](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctions)
-goroutines(Start, Count) | Equivalent to API call [ListGoroutines](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListGoroutines)
+goroutines(Start, Count, Filters, GoroutineGroupingOptions) | Equivalent to API call [ListGoroutines](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListGoroutines)
 local_vars(Scope, Cfg) | Equivalent to API call [ListLocalVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListLocalVars)
 package_vars(Filter, Cfg) | Equivalent to API call [ListPackageVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListPackageVars)
 packages_build_info(IncludeFiles) | Equivalent to API call [ListPackagesBuildInfo](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListPackagesBuildInfo)

_fixtures/goroutinegroup.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"context"
+	"runtime"
+	"runtime/pprof"
+	"sync"
+	"time"
+)
+
+func sleepyfunc(wg *sync.WaitGroup, lbl string) {
+	defer wg.Done()
+	pprof.SetGoroutineLabels(pprof.WithLabels(context.Background(), pprof.Labels("name", lbl)))
+	time.Sleep(10 * 60 * time.Second)
+}
+
+func gopoint1(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func gopoint2(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func gopoint3(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func gopoint4(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func gopoint5(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func startpoint1(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func startpoint2(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func startpoint3(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func startpoint4(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func startpoint5(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func main() {
+	var wg sync.WaitGroup
+
+	for _, lbl := range []string{"one", "two", "three", "four", "five"} {
+		for _, f := range []func(*sync.WaitGroup, string){startpoint1, startpoint2, startpoint3, startpoint4, startpoint5} {
+			for i := 0; i < 1000; i++ {
+				wg.Add(5)
+				gopoint1(&wg, lbl, f)
+				gopoint2(&wg, lbl, f)
+				gopoint3(&wg, lbl, f)
+				gopoint4(&wg, lbl, f)
+				gopoint5(&wg, lbl, f)
+			}
+		}
+	}
+	time.Sleep(2 * time.Second)
+	runtime.Breakpoint()
+	wg.Wait()
+}

cmd/dlv/dlv_test.go

@@ -326,6 +326,8 @@ func TestRedirect(t *testing.T) {
 	cmd.Wait()
 }
 
+const checkAutogenDocLongOutput = false
+
 func checkAutogenDoc(t *testing.T, filename, gencommand string, generated []byte) {
 	saved := slurpFile(t, filepath.Join(projectRoot(), filename))
 
@@ -333,13 +335,17 @@ func checkAutogenDoc(t *testing.T, filename, gencommand string, generated []byte
 	generated = bytes.ReplaceAll(generated, []byte("\r\n"), []byte{'\n'})
 
 	if len(saved) != len(generated) {
-		t.Logf("generated %q saved %q\n", generated, saved)
+		if checkAutogenDocLongOutput {
+			t.Logf("generated %q saved %q\n", generated, saved)
+		}
 		t.Fatalf("%s: needs to be regenerated; run %s", filename, gencommand)
 	}
 
 	for i := range saved {
 		if saved[i] != generated[i] {
-			t.Logf("generated %q saved %q\n", generated, saved)
+			if checkAutogenDocLongOutput {
+				t.Logf("generated %q saved %q\n", generated, saved)
+			}
 			t.Fatalf("%s: needs to be regenerated; run %s", filename, gencommand)
 		}
 	}

pkg/proc/variables.go

@@ -521,6 +521,20 @@ func (g *G) StartLoc() Location {
 	return Location{PC: g.StartPC, File: f, Line: l, Fn: fn}
 }
 
+// System returns true if g is a system goroutine. See isSystemGoroutine in
+// $GOROOT/src/runtime/traceback.go.
+func (g *G) System() bool {
+	loc := g.StartLoc()
+	if loc.Fn == nil {
+		return false
+	}
+	switch loc.Fn.Name {
+	case "runtime.main", "runtime.handleAsyncEvent", "runtime.runfinq":
+		return false
+	}
+	return strings.HasPrefix(loc.Fn.Name, "runtime.")
+}
+
 func (g *G) Labels() map[string]string {
 	if g.labels != nil {
 		return *g.labels

pkg/terminal/command.go

@@ -220,18 +220,62 @@ If called with the linespec argument it will delete all the breakpoints matching
 toggle <breakpoint name or id>`},
 		{aliases: []string{"goroutines", "grs"}, group: goroutineCmds, cmdFn: goroutines, helpMsg: `List program goroutines.
 
-	goroutines [-u (default: user location)|-r (runtime location)|-g (go statement location)|-s (start location)] [-t (stack trace)] [-l (labels)]
+	goroutines [-u|-r|-g|-s] [-t [depth]] [-l] [-with loc expr] [-without loc expr] [-group argument]
 
 Print out info for every goroutine. The flag controls what information is shown along with each goroutine:
 
-	-u	displays location of topmost stackframe in user code
+	-u	displays location of topmost stackframe in user code (default)
 	-r	displays location of topmost stackframe (including frames inside private runtime functions)
 	-g	displays location of go instruction that created the goroutine
 	-s	displays location of the start function
-	-t	displays goroutine's stacktrace
+	-t	displays goroutine's stacktrace (an optional depth value can be specified, default: 10)
 	-l	displays goroutine's labels
 
-If no flag is specified the default is -u.`},
+If no flag is specified the default is -u, i.e. the first frame within the first 30 frames that is not executing a runtime private function.
+
+FILTERING
+
+If -with or -without are specified only goroutines that match the given condition are returned.
+
+To only display goroutines where the specified location contains (or does not contain, for -without and -wo) expr as a substring, use:
+
+	goroutines -with (userloc|curloc|goloc|startloc) expr
+	goroutines -w (userloc|curloc|goloc|startloc) expr
+	goroutines -without (userloc|curloc|goloc|startloc) expr
+	goroutines -wo (userloc|curloc|goloc|startloc) expr
+	
+To only display goroutines that have (or do not have) the specified label key and value, use:
+	
+
+	goroutines -with label key=value
+	goroutines -without label key=value
+	
+To only display goroutines that have (or do not have) the specified label key, use:
+
+	goroutines -with label key
+	goroutines -without label key
+	
+To only display goroutines that are running (or are not running) on a OS thread, use:
+
+
+	goroutines -with running
+	goroutines -without running
+	
+To only display user (or runtime) goroutines, use:
+
+	goroutines -with user
+	goroutines -without user
+
+GROUPING
+
+	goroutines -group (userloc|curloc|goloc|startloc|running|user)
+
+Groups goroutines by the given location, running status or user classification, up to 5 goroutines per group will be displayed as well as the total number of goroutines in the group.
+
+	goroutines -group label key
+
+Groups goroutines by the value of the label with the specified key.
+`},
 		{aliases: []string{"goroutine", "gr"}, group: goroutineCmds, allowedPrefixes: onPrefix, cmdFn: c.goroutine, helpMsg: `Shows or changes current goroutine
 
 	goroutine
@@ -717,67 +761,116 @@ const (
 	printGoroutinesLabels
 )
 
-func printGoroutines(t *Term, gs []*api.Goroutine, fgl formatGoroutineLoc, flags printGoroutinesFlags, state *api.DebuggerState) error {
+func printGoroutines(t *Term, indent string, gs []*api.Goroutine, fgl formatGoroutineLoc, flags printGoroutinesFlags, depth int, state *api.DebuggerState) error {
 	for _, g := range gs {
-		prefix := "  "
+		prefix := indent + "  "
 		if state.SelectedGoroutine != nil && g.ID == state.SelectedGoroutine.ID {
-			prefix = "* "
+			prefix = indent + "* "
 		}
 		fmt.Printf("%sGoroutine %s\n", prefix, t.formatGoroutine(g, fgl))
 		if flags&printGoroutinesLabels != 0 {
-			writeGoroutineLabels(os.Stdout, g, "\t")
+			writeGoroutineLabels(os.Stdout, g, indent+"\t")
 		}
 		if flags&printGoroutinesStack != 0 {
-			stack, err := t.client.Stacktrace(g.ID, 10, 0, nil)
+			stack, err := t.client.Stacktrace(g.ID, depth, 0, nil)
 			if err != nil {
 				return err
 			}
-			printStack(t, os.Stdout, stack, "\t", false)
+			printStack(t, os.Stdout, stack, indent+"\t", false)
 		}
 	}
 	return nil
 }
 
+const (
+	maxGroupMembers    = 5
+	maxGoroutineGroups = 50
+)
+
 func goroutines(t *Term, ctx callContext, argstr string) error {
 	args := strings.Split(argstr, " ")
+	var filters []api.ListGoroutinesFilter
+	var group api.GoroutineGroupingOptions
 	var fgl = fglUserCurrent
 	var flags printGoroutinesFlags
+	var depth = 10
+	var batchSize = goroutineBatchSize
 
-	switch len(args) {
-	case 0:
-		// nothing to do
-	case 1, 2:
-		for _, arg := range args {
-			switch arg {
-			case "-u":
-				fgl = fglUserCurrent
-			case "-r":
-				fgl = fglRuntimeCurrent
-			case "-g":
-				fgl = fglGo
-			case "-s":
-				fgl = fglStart
-			case "-t":
-				flags |= printGoroutinesStack
-			case "-l":
-				flags |= printGoroutinesLabels
-			case "":
-				// nothing to do
-			default:
-				return fmt.Errorf("wrong argument: '%s'", arg)
+	group.MaxGroupMembers = maxGroupMembers
+	group.MaxGroups = maxGoroutineGroups
+
+	for i := 0; i < len(args); i++ {
+		arg := args[i]
+		switch arg {
+		case "-u":
+			fgl = fglUserCurrent
+		case "-r":
+			fgl = fglRuntimeCurrent
+		case "-g":
+			fgl = fglGo
+		case "-s":
+			fgl = fglStart
+		case "-l":
+			flags |= printGoroutinesLabels
+		case "-t":
+			flags |= printGoroutinesStack
+			// optional depth argument
+			if i+1 < len(args) && len(args[i+1]) > 0 {
+				n, err := strconv.Atoi(args[i+1])
+				if err == nil {
+					depth = n
+					i++
+				}
 			}
+
+		case "-w", "-with":
+			filter, err := readGoroutinesFilter(args, &i)
+			if err != nil {
+				return err
+			}
+			filters = append(filters, *filter)
+
+		case "-wo", "-without":
+			filter, err := readGoroutinesFilter(args, &i)
+			if err != nil {
+				return err
+			}
+			filter.Negated = true
+			filters = append(filters, *filter)
+
+		case "-group":
+			var err error
+			group.GroupBy, err = readGoroutinesFilterKind(args, i+1)
+			if err != nil {
+				return err
+			}
+			i++
+			if group.GroupBy == api.GoroutineLabel {
+				if i+1 >= len(args) {
+					return errors.New("-group label must be followed by an argument")
+				}
+				group.GroupByKey = args[i+1]
+				i++
+			}
+			batchSize = 0 // grouping only works well if run on all goroutines
+
+		case "":
+			// nothing to do
+		default:
+			return fmt.Errorf("wrong argument: '%s'", arg)
 		}
-	default:
-		return fmt.Errorf("too many arguments")
 	}
+
 	state, err := t.client.GetState()
 	if err != nil {
 		return err
 	}
 	var (
-		start = 0
-		gslen = 0
-		gs    []*api.Goroutine
+		start         = 0
+		gslen         = 0
+		gs            []*api.Goroutine
+		groups        []api.GoroutineGroup
+		tooManyGroups bool
 	)
 	t.longCommandStart()
 	for start >= 0 {
@@ -785,21 +878,86 @@ func goroutines(t *Term, ctx callContext, argstr string) error {
 			fmt.Printf("interrupted\n")
 			return nil
 		}
-		gs, start, err = t.client.ListGoroutines(start, goroutineBatchSize)
+		gs, groups, start, tooManyGroups, err = t.client.ListGoroutinesWithFilter(start, batchSize, filters, &group)
 		if err != nil {
 			return err
 		}
-		sort.Sort(byGoroutineID(gs))
-		err = printGoroutines(t, gs, fgl, flags, state)
-		if err != nil {
-			return err
+		if len(groups) > 0 {
+			for i := range groups {
+				fmt.Printf("%s\n", groups[i].Name)
+				err = printGoroutines(t, "\t", gs[groups[i].Offset:][:groups[i].Count], fgl, flags, depth, state)
+				if err != nil {
+					return err
+				}
+				fmt.Printf("\tTotal: %d\n", groups[i].Total)
+				if i != len(groups)-1 {
+					fmt.Printf("\n")
+				}
+			}
+			if tooManyGroups {
+				fmt.Printf("Too many groups\n")
+			}
+		} else {
+			sort.Sort(byGoroutineID(gs))
+			err = printGoroutines(t, "", gs, fgl, flags, depth, state)
+			if err != nil {
+				return err
+			}
+			gslen += len(gs)
 		}
-		gslen += len(gs)
 	}
-	fmt.Printf("[%d goroutines]\n", gslen)
+	if gslen > 0 {
+		fmt.Printf("[%d goroutines]\n", gslen)
+	}
 	return nil
 }
 
+func readGoroutinesFilterKind(args []string, i int) (api.GoroutineField, error) {
+	if i >= len(args) {
+		return api.GoroutineFieldNone, fmt.Errorf("%s must be followed by an argument", args[i-1])
+	}
+
+	switch args[i] {
+	case "curloc":
+		return api.GoroutineCurrentLoc, nil
+	case "userloc":
+		return api.GoroutineUserLoc, nil
+	case "goloc":
+		return api.GoroutineGoLoc, nil
+	case "startloc":
+		return api.GoroutineStartLoc, nil
+	case "label":
+		return api.GoroutineLabel, nil
+	case "running":
+		return api.GoroutineRunning, nil
+	case "user":
+		return api.GoroutineUser, nil
+	default:
+		return api.GoroutineFieldNone, fmt.Errorf("unrecognized argument to %s %s", args[i-1], args[i])
+	}
+}
+
+func readGoroutinesFilter(args []string, pi *int) (*api.ListGoroutinesFilter, error) {
+	r := new(api.ListGoroutinesFilter)
+	var err error
+	r.Kind, err = readGoroutinesFilterKind(args, *pi+1)
+	if err != nil {
+		return nil, err
+	}
+	*pi++
+	switch r.Kind {
+	case api.GoroutineRunning, api.GoroutineUser:
+		return r, nil
+	}
+	if *pi+1 >= len(args) {
+		return nil, fmt.Errorf("%s %s needs to be followed by an expression", args[*pi-1], args[*pi])
+	}
+	r.Arg = args[*pi+1]
+	*pi++
+
+	return r, nil
+}
+
 func selectedGID(state *api.DebuggerState) int {
 	if state.SelectedGoroutine == nil {
 		return 0

pkg/terminal/starbind/starlark_mapping.go

@@ -912,6 +912,18 @@ func (env *Env) starlarkPredeclare() starlark.StringDict {
 				return starlark.None, decorateError(thread, err)
 			}
 		}
+		if len(args) > 2 && args[2] != starlark.None {
+			err := unmarshalStarlarkValue(args[2], &rpcArgs.Filters, "Filters")
+			if err != nil {
+				return starlark.None, decorateError(thread, err)
+			}
+		}
+		if len(args) > 3 && args[3] != starlark.None {
+			err := unmarshalStarlarkValue(args[3], &rpcArgs.GoroutineGroupingOptions, "GoroutineGroupingOptions")
+			if err != nil {
+				return starlark.None, decorateError(thread, err)
+			}
+		}
 		for _, kv := range kwargs {
 			var err error
 			switch kv[0].(starlark.String) {
@@ -919,6 +931,10 @@ func (env *Env) starlarkPredeclare() starlark.StringDict {
 				err = unmarshalStarlarkValue(kv[1], &rpcArgs.Start, "Start")
 			case "Count":
 				err = unmarshalStarlarkValue(kv[1], &rpcArgs.Count, "Count")
+			case "Filters":
+				err = unmarshalStarlarkValue(kv[1], &rpcArgs.Filters, "Filters")
+			case "GoroutineGroupingOptions":
+				err = unmarshalStarlarkValue(kv[1], &rpcArgs.GoroutineGroupingOptions, "GoroutineGroupingOptions")
 			default:
 				err = fmt.Errorf("unknown argument %q", kv[0])
 			}

service/api/types.go

@@ -586,3 +586,41 @@ type DumpState struct {
 
 	Err string
 }
+
+// ListGoroutinesFilter describes a filtering condition for the
+// ListGoroutines API call.
+type ListGoroutinesFilter struct {
+	Kind    GoroutineField
+	Negated bool
+	Arg     string
+}
+
+// GoroutineField allows referring to a field of a goroutine object.
+type GoroutineField uint8
+
+const (
+	GoroutineFieldNone  GoroutineField = iota
+	GoroutineCurrentLoc                // the goroutine's CurrentLoc
+	GoroutineUserLoc                   // the goroutine's UserLoc
+	GoroutineGoLoc                     // the goroutine's GoStatementLoc
+	GoroutineStartLoc                  // the goroutine's StartLoc
+	GoroutineLabel                     // the goroutine's label
+	GoroutineRunning                   // the goroutine is running
+	GoroutineUser                      // the goroutine is a user goroutine
+)
+
+// GoroutineGroup represents a group of goroutines in the return value of
+// the ListGoroutines API call.
+type GoroutineGroup struct {
+	Name   string // name of this group
+	Offset int    // start offset in the list of goroutines of this group
+	Count  int    // number of goroutines that belong to this group in the list of goroutines
+	Total  int    // total number of goroutines that belong to this group
+}
+
+type GoroutineGroupingOptions struct {
+	GroupBy         GoroutineField
+	GroupByKey      string
+	MaxGroupMembers int
+	MaxGroups       int
+}

service/client.go

@@ -114,6 +114,8 @@ type Client interface {
 
 	// ListGoroutines lists all goroutines.
 	ListGoroutines(start, count int) ([]*api.Goroutine, int, error)
+	// ListGoroutinesWithFilter lists goroutines matching the filters
+	ListGoroutinesWithFilter(start, count int, filters []api.ListGoroutinesFilter, group *api.GoroutineGroupingOptions) ([]*api.Goroutine, []api.GoroutineGroup, int, bool, error)
 
 	// Returns stacktrace
 	Stacktrace(goroutineID int, depth int, opts api.StacktraceOptions, cfg *api.LoadConfig) ([]api.Stackframe, error)

service/dap/server.go

@@ -1550,12 +1550,9 @@ func (s *Server) onStackTraceRequest(request *dap.StackTraceRequest) {
 	}
 
 	// Determine if the goroutine is a system goroutine.
-	// TODO(suzmue): Use the System() method defined in: https://github.com/go-delve/delve/pull/2504
-	g, err := s.debugger.FindGoroutine(goroutineID)
-	var isSystemGoroutine bool
-	if err == nil {
-		userLoc := g.UserCurrent()
-		isSystemGoroutine = fnPackageName(&userLoc) == "runtime"
+	isSystemGoroutine := true
+	if g, _ := s.debugger.FindGoroutine(goroutineID); g != nil {
+		isSystemGoroutine = g.System()
 	}
 
 	stackFrames := make([]dap.StackFrame, len(frames))

service/debugger/debugger.go

@@ -1525,6 +1525,131 @@ func (d *Debugger) Goroutines(start, count int) ([]*proc.G, int, error) {
 	return proc.GoroutinesInfo(d.target, start, count)
 }
 
+// FilterGoroutines returns the goroutines in gs that satisfy the specified filters.
+func (d *Debugger) FilterGoroutines(gs []*proc.G, filters []api.ListGoroutinesFilter) []*proc.G {
+	if len(filters) == 0 {
+		return gs
+	}
+	d.targetMutex.Lock()
+	defer d.targetMutex.Unlock()
+	r := []*proc.G{}
+	for _, g := range gs {
+		ok := true
+		for i := range filters {
+			if !matchGoroutineFilter(g, &filters[i]) {
+				ok = false
+				break
+			}
+		}
+		if ok {
+			r = append(r, g)
+		}
+	}
+	return r
+}
+
+func matchGoroutineFilter(g *proc.G, filter *api.ListGoroutinesFilter) bool {
+	var val bool
+	switch filter.Kind {
+	default:
+		fallthrough
+	case api.GoroutineFieldNone:
+		val = true
+	case api.GoroutineCurrentLoc:
+		val = matchGoroutineLocFilter(g.CurrentLoc, filter.Arg)
+	case api.GoroutineUserLoc:
+		val = matchGoroutineLocFilter(g.UserCurrent(), filter.Arg)
+	case api.GoroutineGoLoc:
+		val = matchGoroutineLocFilter(g.Go(), filter.Arg)
+	case api.GoroutineStartLoc:
+		val = matchGoroutineLocFilter(g.StartLoc(), filter.Arg)
+	case api.GoroutineLabel:
+		idx := strings.Index(filter.Arg, "=")
+		if idx >= 0 {
+			val = g.Labels()[filter.Arg[:idx]] == filter.Arg[idx+1:]
+		} else {
+			_, val = g.Labels()[filter.Arg]
+		}
+	case api.GoroutineRunning:
+		val = g.Thread != nil
+	case api.GoroutineUser:
+		val = !g.System()
+	}
+	if filter.Negated {
+		val = !val
+	}
+	return val
+}
+
+func matchGoroutineLocFilter(loc proc.Location, arg string) bool {
+	return strings.Contains(formatLoc(loc), arg)
+}
+
+func formatLoc(loc proc.Location) string {
+	fnname := "?"
+	if loc.Fn != nil {
+		fnname = loc.Fn.Name
+	}
+	return fmt.Sprintf("%s:%d in %s", loc.File, loc.Line, fnname)
+}
+
+// GroupGoroutines divides goroutines in gs into groups as specified by groupBy and groupByArg.
+// A maximum of maxGoroutinesPerGroup are saved in each group, but the total
+// number of goroutines in each group is recorded.
+func (d *Debugger) GroupGoroutines(gs []*proc.G, group *api.GoroutineGroupingOptions) ([]*proc.G, []api.GoroutineGroup, bool) {
+	if group.GroupBy == api.GoroutineFieldNone {
+		return gs, nil, false
+	}
+	d.targetMutex.Lock()
+	defer d.targetMutex.Unlock()
+
+	groupMembers := map[string][]*proc.G{}
+	totals := map[string]int{}
+
+	for _, g := range gs {
+		var key string
+		switch group.GroupBy {
+		case api.GoroutineCurrentLoc:
+			key = formatLoc(g.CurrentLoc)
+		case api.GoroutineUserLoc:
+			key = formatLoc(g.UserCurrent())
+		case api.GoroutineGoLoc:
+			key = formatLoc(g.Go())
+		case api.GoroutineStartLoc:
+			key = formatLoc(g.StartLoc())
+		case api.GoroutineLabel:
+			key = fmt.Sprintf("%s=%s", group.GroupByKey, g.Labels()[group.GroupByKey])
+		case api.GoroutineRunning:
+			key = fmt.Sprintf("running=%v", g.Thread != nil)
+		case api.GoroutineUser:
+			key = fmt.Sprintf("user=%v", !g.System())
+		}
+		if len(groupMembers[key]) < group.MaxGroupMembers {
+			groupMembers[key] = append(groupMembers[key], g)
+		}
+		totals[key]++
+	}
+
+	keys := make([]string, 0, len(groupMembers))
+	for key := range groupMembers {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	tooManyGroups := false
+	gsout := []*proc.G{}
+	groups := []api.GoroutineGroup{}
+	for _, key := range keys {
+		if group.MaxGroups > 0 && len(groups) >= group.MaxGroups {
+			tooManyGroups = true
+			break
+		}
+		groups = append(groups, api.GoroutineGroup{Name: key, Offset: len(gsout), Count: len(groupMembers[key]), Total: totals[key]})
+		gsout = append(gsout, groupMembers[key]...)
+	}
+	return gsout, groups, tooManyGroups
+}
+
 // Stacktrace returns a list of Stackframes for the given goroutine. The
 // length of the returned list will be min(stack_len, depth).
 // If 'full' is true, then local vars, function args, etc will be returned as well.

service/rpc2/client.go

@@ -356,10 +356,19 @@ func (c *RPCClient) ListFunctionArgs(scope api.EvalScope, cfg api.LoadConfig) ([
 
 func (c *RPCClient) ListGoroutines(start, count int) ([]*api.Goroutine, int, error) {
 	var out ListGoroutinesOut
-	err := c.call("ListGoroutines", ListGoroutinesIn{start, count}, &out)
+	err := c.call("ListGoroutines", ListGoroutinesIn{start, count, nil, api.GoroutineGroupingOptions{}}, &out)
 	return out.Goroutines, out.Nextg, err
 }
 
+func (c *RPCClient) ListGoroutinesWithFilter(start, count int, filters []api.ListGoroutinesFilter, group *api.GoroutineGroupingOptions) ([]*api.Goroutine, []api.GoroutineGroup, int, bool, error) {
+	if group == nil {
+		group = &api.GoroutineGroupingOptions{}
+	}
+	var out ListGoroutinesOut
+	err := c.call("ListGoroutines", ListGoroutinesIn{start, count, filters, *group}, &out)
+	return out.Goroutines, out.Groups, out.Nextg, out.TooManyGroups, err
+}
+
 func (c *RPCClient) Stacktrace(goroutineId, depth int, opts api.StacktraceOptions, cfg *api.LoadConfig) ([]api.Stackframe, error) {
 	var out StacktraceOut
 	err := c.call("Stacktrace", StacktraceIn{goroutineId, depth, false, false, opts, cfg}, &out)

service/rpc2/server.go

@@ -585,11 +585,16 @@ func (s *RPCServer) ListTypes(arg ListTypesIn, out *ListTypesOut) error {
 type ListGoroutinesIn struct {
 	Start int
 	Count int
+
+	Filters []api.ListGoroutinesFilter
+	api.GoroutineGroupingOptions
 }
 
 type ListGoroutinesOut struct {
-	Goroutines []*api.Goroutine
-	Nextg      int
+	Goroutines    []*api.Goroutine
+	Nextg         int
+	Groups        []api.GoroutineGroup
+	TooManyGroups bool
 }
 
 // ListGoroutines lists all goroutines.
@@ -598,11 +603,37 @@ type ListGoroutinesOut struct {
 // parameter, to get more goroutines from ListGoroutines.
 // Passing a value of Start that wasn't returned by ListGoroutines will skip
 // an undefined number of goroutines.
+//
+// If arg.Filters are specified the list of returned goroutines is filtered
+// applying the specified filters.
+// For example:
+//    ListGoroutinesFilter{ Kind: ListGoroutinesFilterUserLoc, Negated: false, Arg: "afile.go" }
+// will only return goroutines whose UserLoc contains "afile.go" as a substring.
+// More specifically a goroutine matches a location filter if the specified
+// location, formatted like this:
+//    filename:lineno in function
+// contains Arg[0] as a substring.
+//
+// Filters can also be applied to goroutine labels:
+//    ListGoroutineFilter{ Kind: ListGoroutinesFilterLabel, Negated: false, Arg: "key=value" }
+// this filter will only return goroutines that have a key=value label.
+//
+// If arg.GroupBy is not GoroutineFieldNone then the goroutines will
+// be grouped with the specified criterion.
+// If the value of arg.GroupBy is GoroutineLabel goroutines will
+// be grouped by the value of the label with key GroupByKey.
+// For each group a maximum of MaxExamples example goroutines are
+// returned, as well as the total number of goroutines in the group.
 func (s *RPCServer) ListGoroutines(arg ListGoroutinesIn, out *ListGoroutinesOut) error {
+	//TODO(aarzilli): if arg contains a running goroutines filter (not negated)
+	// and start == 0 and count == 0 then we can optimize this by just looking
+	// at threads directly.
 	gs, nextg, err := s.debugger.Goroutines(arg.Start, arg.Count)
 	if err != nil {
 		return err
 	}
+	gs = s.debugger.FilterGoroutines(gs, arg.Filters)
+	gs, out.Groups, out.TooManyGroups = s.debugger.GroupGoroutines(gs, &arg.GoroutineGroupingOptions)
 	s.debugger.LockTarget()
 	defer s.debugger.UnlockTarget()
 	out.Goroutines = api.ConvertGoroutines(gs)

service/test/integration2_test.go

@@ -2391,3 +2391,29 @@ func TestStopServerWithClosedListener(t *testing.T) {
 	listener.Close()
 	time.Sleep(1 * time.Second) // give time to server to panic
 }
+
+func TestGoroutinesGrouping(t *testing.T) {
+	// Tests the goroutine grouping and filtering feature
+	withTestClient2("goroutinegroup", t, func(c service.Client) {
+		state := <-c.Continue()
+		assertNoError(state.Err, t, "Continue")
+		_, ggrp, _, _, err := c.ListGoroutinesWithFilter(0, 0, nil, &api.GoroutineGroupingOptions{GroupBy: api.GoroutineLabel, GroupByKey: "name", MaxGroupMembers: 5, MaxGroups: 10})
+		assertNoError(err, t, "ListGoroutinesWithFilter (group by label)")
+		t.Logf("%#v\n", ggrp)
+		if len(ggrp) < 5 {
+			t.Errorf("not enough groups %d\n", len(ggrp))
+		}
+		var unnamedCount int
+		for i := range ggrp {
+			if ggrp[i].Name == "name=" {
+				unnamedCount = ggrp[i].Total
+				break
+			}
+		}
+		gs, _, _, _, err := c.ListGoroutinesWithFilter(0, 0, []api.ListGoroutinesFilter{{Kind: api.GoroutineLabel, Arg: "name="}}, nil)
+		assertNoError(err, t, "ListGoroutinesWithFilter (filter unnamed)")
+		if len(gs) != unnamedCount {
+			t.Errorf("wrong number of goroutines returned by filter: %d (expected %d)\n", len(gs), unnamedCount)
+		}
+	})
+}