proc,terminal,service: support stack watchpoints (#2521)

* terminal,service: add way to see internal breakpoints

Now that Delve has internal breakpoints that survive for long periods
of time it will be useful to have an option to display them.

* proc,terminal,service: support stack watchpoints

Adds support for watchpoints on stack allocated variables.

When a stack variable is watched, in addition to the normal watchpoint
some support breakpoints are created:

- one breakpoint inside runtime.copystack, used to adjust the address
  of the watchpoint when the stack is resized
- one or more breakpoints used to detect when the stack variable goes
  out of scope, those are similar to the breakpoints set by StepOut.

Implements #279

Documentation/backend_test_health.md

@@ -1,31 +1,31 @@
 Tests skipped by each supported backend:
 
-* 386 skipped = 6
+* 386 skipped = 7
 	* 1 broken
 	* 3 broken - cgo stacktraces
-	* 2 not implemented
-* arm64 skipped = 4
+	* 3 not implemented
+* arm64 skipped = 5
 	* 1 broken
 	* 1 broken - global variable symbolication
-	* 2 not implemented
-* darwin skipped = 2
-	* 2 not implemented
+	* 3 not implemented
+* darwin skipped = 3
+	* 3 not implemented
 * darwin/arm64 skipped = 1
 	* 1 broken - cgo stacktraces
 * darwin/lldb skipped = 1
 	* 1 upstream issue
-* freebsd skipped = 14
+* freebsd skipped = 15
 	* 11 broken
-	* 3 not implemented
+	* 4 not implemented
 * linux/386/pie skipped = 1
 	* 1 broken
 * linux/arm64 skipped = 1
 	* 1 broken - cgo stacktraces
 * pie skipped = 2
 	* 2 upstream issue - https://github.com/golang/go/issues/29322
-* rr skipped = 2
-	* 2 not implemented
-* windows skipped = 4
+* rr skipped = 3
+	* 3 not implemented
+* windows skipped = 5
 	* 1 broken
-	* 2 not implemented
+	* 3 not implemented
 	* 1 upstream issue

Documentation/cli/README.md

@@ -114,6 +114,10 @@ Aliases: b
 
 ## breakpoints
 Print out info for active breakpoints.
+	
+	breakpoints [-a]
+
+Specifying -a prints all physical breakpoint, including internal breakpoints.
 
 Aliases: bp
 

Documentation/cli/starlark.md

@@ -42,7 +42,7 @@ get_buffered_tracepoints() | Equivalent to API call [GetBufferedTracepoints](htt
 get_thread(Id) | Equivalent to API call [GetThread](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.GetThread)
 is_multiclient() | Equivalent to API call [IsMulticlient](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.IsMulticlient)
 last_modified() | Equivalent to API call [LastModified](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.LastModified)
-breakpoints() | Equivalent to API call [ListBreakpoints](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListBreakpoints)
+breakpoints(All) | Equivalent to API call [ListBreakpoints](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListBreakpoints)
 checkpoints() | Equivalent to API call [ListCheckpoints](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListCheckpoints)
 dynamic_libraries() | Equivalent to API call [ListDynamicLibraries](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListDynamicLibraries)
 function_args(Scope, Cfg) | Equivalent to API call [ListFunctionArgs](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctionArgs)

_fixtures/databpstack.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"runtime"
+)
+
+func f() {
+	w := 0
+	runtime.Breakpoint()
+	g(1000, &w) // Position 0
+}
+
+func g(cnt int, p *int) {
+	if cnt == 0 {
+		*p = 10
+		return // Position 1
+	}
+	g(cnt-1, p)
+}
+
+func main() {
+	f()
+	fmt.Printf("done\n") // Position 2
+}

pkg/proc/breakpoints.go

@@ -42,9 +42,10 @@ type Breakpoint struct {
 	Name         string // User defined name of the breakpoint
 	LogicalID    int    // ID of the logical breakpoint that owns this physical breakpoint
 
-	WatchExpr    string
-	WatchType    WatchType
-	HWBreakIndex uint8 // hardware breakpoint index
+	WatchExpr     string
+	WatchType     WatchType
+	HWBreakIndex  uint8 // hardware breakpoint index
+	watchStackOff int64 // for watchpoints of stack variables, offset of the address from top of the stack
 
 	// Breaklets is the list of overlapping breakpoints on this physical breakpoint.
 	// There can be at most one UserBreakpoint in this list but multiple internal breakpoints are allowed.
@@ -94,6 +95,20 @@ type Breaklet struct {
 		Op  token.Token
 		Val int
 	}
+
+	// checkPanicCall checks that the breakpoint happened while the function was
+	// called by a panic. It is only checked for WatchOutOfScopeBreakpoint Kind.
+	checkPanicCall bool
+
+	// callback is called if every other condition for this breaklet is met,
+	// the return value will determine if the breaklet should be considered
+	// active.
+	// The callback can have side-effects.
+	callback func(th Thread) bool
+
+	// For WatchOutOfScopeBreakpoints and StackResizeBreakpoints the watchpoint
+	// field contains the watchpoint related to this out of scope sentinel.
+	watchpoint *Breakpoint
 }
 
 // BreakpointKind determines the behavior of delve when the
@@ -116,6 +131,14 @@ const (
 	// destination of CALL, delete this breakpoint and then continue again
 	StepBreakpoint
 
+	// WatchOutOfScopeBreakpoint is a breakpoint used to detect when a watched
+	// stack variable goes out of scope.
+	WatchOutOfScopeBreakpoint
+
+	// StackResizeBreakpoint is a breakpoint used to detect stack resizes to
+	// adjust the watchpoint of stack variables.
+	StackResizeBreakpoint
+
 	steppingMask = NextBreakpoint | NextDeferBreakpoint | StepBreakpoint
 )
 
@@ -153,6 +176,38 @@ func (bp *Breakpoint) String() string {
 	return fmt.Sprintf("Breakpoint %d at %#v %s:%d", bp.LogicalID, bp.Addr, bp.File, bp.Line)
 }
 
+// VerboseDescr returns a string describing parts of the breakpoint struct
+// that aren't otherwise user visible, for debugging purposes.
+func (bp *Breakpoint) VerboseDescr() []string {
+	r := []string{}
+
+	r = append(r, fmt.Sprintf("OriginalData=%#x", bp.OriginalData))
+
+	if bp.WatchType != 0 {
+		r = append(r, fmt.Sprintf("HWBreakIndex=%#x watchStackOff=%#x", bp.HWBreakIndex, bp.watchStackOff))
+	}
+
+	for _, breaklet := range bp.Breaklets {
+		switch breaklet.Kind {
+		case UserBreakpoint:
+			r = append(r, fmt.Sprintf("User Cond=%q HitCond=%v", exprToString(breaklet.Cond), breaklet.HitCond))
+		case NextBreakpoint:
+			r = append(r, fmt.Sprintf("Next Cond=%q", exprToString(breaklet.Cond)))
+		case NextDeferBreakpoint:
+			r = append(r, fmt.Sprintf("NextDefer Cond=%q DeferReturns=%#x", exprToString(breaklet.Cond), breaklet.DeferReturns))
+		case StepBreakpoint:
+			r = append(r, fmt.Sprintf("Step Cond=%q", exprToString(breaklet.Cond)))
+		case WatchOutOfScopeBreakpoint:
+			r = append(r, fmt.Sprintf("WatchOutOfScope Cond=%q checkPanicCall=%v", exprToString(breaklet.Cond), breaklet.checkPanicCall))
+		case StackResizeBreakpoint:
+			r = append(r, fmt.Sprintf("StackResizeBreakpoint Cond=%q", exprToString(breaklet.Cond)))
+		default:
+			r = append(r, fmt.Sprintf("Unknown %d", breaklet.Kind))
+		}
+	}
+	return r
+}
+
 // BreakpointExistsError is returned when trying to set a breakpoint at
 // an address that already has a breakpoint set for it.
 type BreakpointExistsError struct {
@@ -232,12 +287,27 @@ func (bpstate *BreakpointState) checkCond(tgt *Target, breaklet *Breaklet, threa
 			}
 		}
 
+	case WatchOutOfScopeBreakpoint:
+		if breaklet.checkPanicCall {
+			frames, err := ThreadStacktrace(thread, 2)
+			if err == nil {
+				ipc, _ := isPanicCall(frames)
+				active = active && ipc
+			}
+		}
+
+	case StackResizeBreakpoint:
+		// no further checks
+
 	default:
 		bpstate.CondError = fmt.Errorf("internal error unknown breakpoint kind %v", breaklet.Kind)
 	}
 
 	if active {
-		bpstate.Active = true
+		if breaklet.callback != nil {
+			active = breaklet.callback(thread)
+		}
+		bpstate.Active = active
 	}
 }
 
@@ -372,6 +442,10 @@ func (nbp NoBreakpointError) Error() string {
 type BreakpointMap struct {
 	M map[uint64]*Breakpoint
 
+	// WatchOutOfScope is the list of watchpoints that went out of scope during
+	// the last resume operation
+	WatchOutOfScope []*Breakpoint
+
 	breakpointIDCounter         int
 	internalBreakpointIDCounter int
 }
@@ -466,16 +540,31 @@ func (t *Target) SetWatchpoint(scope *EvalScope, expr string, wtype WatchType, c
 		//member fields here.
 		return nil, fmt.Errorf("can not watch variable of type %s", xv.DwarfType.String())
 	}
-	if xv.Addr >= scope.g.stack.lo && xv.Addr < scope.g.stack.hi {
-		//TODO(aarzilli): support watching stack variables
-		return nil, errors.New("can not watch stack allocated variable")
+
+	stackWatch := scope.g != nil && !scope.g.SystemStack && xv.Addr >= scope.g.stack.lo && xv.Addr < scope.g.stack.hi
+
+	if stackWatch && wtype&WatchRead != 0 {
+		// In theory this would work except for the fact that the runtime will
+		// read them randomly to resize stacks so it doesn't make sense to do
+		// this.
+		return nil, errors.New("can not watch stack allocated variable for reads")
 	}
 
 	bp, err := t.setBreakpointInternal(xv.Addr, UserBreakpoint, wtype.withSize(uint8(sz)), cond)
-	if bp != nil {
-		bp.WatchExpr = expr
+	if err != nil {
+		return bp, err
 	}
-	return bp, err
+	bp.WatchExpr = expr
+
+	if stackWatch {
+		bp.watchStackOff = int64(bp.Addr) - int64(scope.g.stack.hi)
+		err := t.setStackWatchBreakpoints(scope, bp)
+		if err != nil {
+			return bp, err
+		}
+	}
+
+	return bp, nil
 }
 
 func (t *Target) setBreakpointInternal(addr uint64, kind BreakpointKind, wtype WatchType, cond ast.Expr) (*Breakpoint, error) {
@@ -588,6 +677,15 @@ func (t *Target) ClearBreakpoint(addr uint64) (*Breakpoint, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	if bp.WatchExpr != "" && bp.watchStackOff != 0 {
+		// stack watchpoint, must remove all its WatchOutOfScopeBreakpoints/StackResizeBreakpoints
+		err := t.clearStackWatchBreakpoints(bp)
+		if err != nil {
+			return bp, err
+		}
+	}
+
 	return bp, nil
 }
 

pkg/proc/proc_test.go

@@ -5448,3 +5448,72 @@ func TestDwrapStartLocation(t *testing.T) {
 		}
 	})
 }
+
+func TestWatchpointStack(t *testing.T) {
+	skipOn(t, "not implemented", "windows")
+	skipOn(t, "not implemented", "freebsd")
+	skipOn(t, "not implemented", "darwin")
+	skipOn(t, "not implemented", "386")
+	skipOn(t, "not implemented", "arm64")
+	skipOn(t, "not implemented", "rr")
+
+	withTestProcess("databpstack", t, func(p *proc.Target, fixture protest.Fixture) {
+		clearlen := len(p.Breakpoints().M)
+
+		assertNoError(p.Continue(), t, "Continue 0")
+		assertLineNumber(p, t, 11, "Continue 0") // Position 0
+
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
+		assertNoError(err, t, "GoroutineScope")
+
+		_, err = p.SetWatchpoint(scope, "w", proc.WatchWrite, nil)
+		assertNoError(err, t, "SetDataBreakpoint(write-only)")
+
+		if len(p.Breakpoints().M) != clearlen+3 {
+			// want 1 watchpoint, 1 stack resize breakpoint, 1 out of scope sentinel
+			t.Errorf("wrong number of breakpoints after setting watchpoint: %d", len(p.Breakpoints().M)-clearlen)
+		}
+
+		var retaddr uint64
+		for _, bp := range p.Breakpoints().M {
+			for _, breaklet := range bp.Breaklets {
+				if breaklet.Kind&proc.WatchOutOfScopeBreakpoint != 0 {
+					retaddr = bp.Addr
+					break
+				}
+			}
+		}
+
+		_, err = p.SetBreakpoint(retaddr, proc.UserBreakpoint, nil)
+		assertNoError(err, t, "SetBreakpoint")
+
+		if len(p.Breakpoints().M) != clearlen+3 {
+			// want 1 watchpoint, 1 stack resize breakpoint, 1 out of scope sentinel (which is also a user breakpoint)
+			t.Errorf("wrong number of breakpoints after setting watchpoint: %d", len(p.Breakpoints().M)-clearlen)
+		}
+
+		assertNoError(p.Continue(), t, "Continue 1")
+		assertLineNumber(p, t, 17, "Continue 1") // Position 1
+
+		assertNoError(p.Continue(), t, "Continue 2")
+		t.Logf("%#v", p.CurrentThread().Breakpoint().Breakpoint)
+		assertLineNumber(p, t, 24, "Continue 2") // Position 2 (watchpoint gone out of scope)
+
+		if len(p.Breakpoints().M) != clearlen+1 {
+			// want 1 user breakpoint set at retaddr
+			t.Errorf("wrong number of breakpoints after watchpoint goes out of scope: %d", len(p.Breakpoints().M)-clearlen)
+		}
+
+		if len(p.Breakpoints().WatchOutOfScope) != 1 {
+			t.Errorf("wrong number of out-of-scope watchpoints after watchpoint goes out of scope: %d", len(p.Breakpoints().WatchOutOfScope))
+		}
+
+		_, err = p.ClearBreakpoint(retaddr)
+		assertNoError(err, t, "ClearBreakpoint")
+
+		if len(p.Breakpoints().M) != clearlen {
+			// want 1 user breakpoint set at retaddr
+			t.Errorf("wrong number of breakpoints after removing user breakpoint: %d", len(p.Breakpoints().M)-clearlen)
+		}
+	})
+}

pkg/proc/stackwatch.go

@@ -0,0 +1,170 @@
+package proc
+
+import (
+	"errors"
+
+	"github.com/go-delve/delve/pkg/astutil"
+	"github.com/go-delve/delve/pkg/logflags"
+)
+
+// This file implements most of the details needed to support stack
+// watchpoints. Some of the remaining details are in breakpoints, along with
+// the code to support non-stack allocated watchpoints.
+//
+// In Go goroutine stacks start small and are frequently resized by the
+// runtime according to the needs of the goroutine.
+// To support this behavior we create a StackResizeBreakpoint, deep inside
+// the Go runtime, when this breakpoint is hit all the stack watchpoints on
+// the goroutine being resized are adjusted for the new stack.
+// Furthermore, we need to detect when a goroutine leaves the stack frame
+// where the variable we are watching was declared, so that we can notify
+// the user that the variable went out of scope and clear the watchpoint.
+//
+// These breakpoints are created by setStackWatchBreakpoints and cleared by
+// clearStackWatchBreakpoints.
+
+// setStackWatchBreakpoints sets the out of scope sentinel breakpoints for
+// watchpoint and a stack resize breakpoint.
+func (t *Target) setStackWatchBreakpoints(scope *EvalScope, watchpoint *Breakpoint) error {
+	// Watchpoint Out-of-scope Sentinel
+
+	woos := func(_ Thread) bool {
+		watchpointOutOfScope(t, watchpoint)
+		return true
+	}
+
+	topframe, retframe, err := topframe(scope.g, nil)
+	if err != nil {
+		return err
+	}
+
+	sameGCond := sameGoroutineCondition(scope.g)
+	retFrameCond := astutil.And(sameGCond, frameoffCondition(&retframe))
+
+	var deferpc uint64
+	if topframe.TopmostDefer != nil {
+		_, _, deferfn := topframe.TopmostDefer.DeferredFunc(t)
+		if deferfn != nil {
+			var err error
+			deferpc, err = FirstPCAfterPrologue(t, deferfn, false)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	if deferpc != 0 && deferpc != topframe.Current.PC {
+		deferbp, err := t.SetBreakpoint(deferpc, WatchOutOfScopeBreakpoint, sameGCond)
+		if err != nil {
+			return err
+		}
+		deferbreaklet := deferbp.Breaklets[len(deferbp.Breaklets)-1]
+		deferbreaklet.checkPanicCall = true
+		deferbreaklet.watchpoint = watchpoint
+		deferbreaklet.callback = woos
+	}
+
+	retbp, err := t.SetBreakpoint(retframe.Current.PC, WatchOutOfScopeBreakpoint, retFrameCond)
+	if err != nil {
+		return err
+	}
+
+	retbreaklet := retbp.Breaklets[len(retbp.Breaklets)-1]
+	retbreaklet.watchpoint = watchpoint
+	retbreaklet.callback = woos
+
+	// Stack Resize Sentinel
+
+	fn := t.BinInfo().LookupFunc["runtime.copystack"]
+	if fn == nil {
+		return errors.New("could not find runtime.copystack")
+	}
+	text, err := Disassemble(t.Memory(), nil, t.Breakpoints(), t.BinInfo(), fn.Entry, fn.End)
+	if err != nil {
+		return err
+	}
+	var retpc uint64
+	for _, instr := range text {
+		if instr.IsRet() {
+			if retpc != 0 {
+				return errors.New("runtime.copystack has too many return instructions")
+			}
+			retpc = instr.Loc.PC
+		}
+	}
+	if retpc == 0 {
+		return errors.New("could not find return instruction in runtime.copystack")
+	}
+	rszbp, err := t.SetBreakpoint(retpc, StackResizeBreakpoint, sameGCond)
+	if err != nil {
+		return err
+	}
+
+	rszbreaklet := rszbp.Breaklets[len(rszbp.Breaklets)-1]
+	rszbreaklet.watchpoint = watchpoint
+	rszbreaklet.callback = func(th Thread) bool {
+		adjustStackWatchpoint(t, th, watchpoint)
+		return false // we never want this breakpoint to be shown to the user
+	}
+
+	return nil
+}
+
+// clearStackWatchBreakpoints clears all accessory breakpoints for
+// watchpoint.
+func (t *Target) clearStackWatchBreakpoints(watchpoint *Breakpoint) error {
+	bpmap := t.Breakpoints()
+	for _, bp := range bpmap.M {
+		changed := false
+		for i, breaklet := range bp.Breaklets {
+			if breaklet.watchpoint == watchpoint {
+				bp.Breaklets[i] = nil
+				changed = true
+			}
+		}
+		if changed {
+			_, err := t.finishClearBreakpoint(bp)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// watchpointOutOfScope is called when the watchpoint goes out of scope. It
+// is used as a breaklet callback function.
+// Its responsibility is to delete the watchpoint and make sure that the
+// user is notified of the watchpoint going out of scope.
+func watchpointOutOfScope(t *Target, watchpoint *Breakpoint) {
+	t.Breakpoints().WatchOutOfScope = append(t.Breakpoints().WatchOutOfScope, watchpoint)
+	_, err := t.ClearBreakpoint(watchpoint.Addr)
+	if err != nil {
+		log := logflags.DebuggerLogger()
+		log.Errorf("could not clear out-of-scope watchpoint: %v", err)
+	}
+}
+
+// adjustStackWatchpoint is called when the goroutine of watchpoint resizes
+// its stack. It is used as a breaklet callback function.
+// Its responsibility is to move the watchpoint to a its new address.
+func adjustStackWatchpoint(t *Target, th Thread, watchpoint *Breakpoint) {
+	g, _ := GetG(th)
+	if g == nil {
+		return
+	}
+	err := t.proc.EraseBreakpoint(watchpoint)
+	if err != nil {
+		log := logflags.DebuggerLogger()
+		log.Errorf("could not adjust watchpoint at %#x: %v", watchpoint.Addr, err)
+		return
+	}
+	delete(t.Breakpoints().M, watchpoint.Addr)
+	watchpoint.Addr = uint64(int64(g.stack.hi) + watchpoint.watchStackOff)
+	err = t.proc.WriteBreakpoint(watchpoint)
+	if err != nil {
+		log := logflags.DebuggerLogger()
+		log.Errorf("could not adjust watchpoint at %#x: %v", watchpoint.Addr, err)
+		return
+	}
+	t.Breakpoints().M[watchpoint.Addr] = watchpoint
+}

pkg/proc/target_exec.go

@@ -54,6 +54,7 @@ func (dbp *Target) Continue() error {
 		thread.Common().CallReturn = false
 		thread.Common().returnValues = nil
 	}
+	dbp.Breakpoints().WatchOutOfScope = nil
 	dbp.CheckAndClearManualStopRequest()
 	defer func() {
 		// Make sure we clear internal breakpoints if we simultaneously receive a
@@ -916,10 +917,12 @@ func setDeferBreakpoint(p *Target, text []AsmInstruction, topframe Stackframe, s
 	var deferpc uint64
 	if topframe.TopmostDefer != nil && topframe.TopmostDefer.DwrapPC != 0 {
 		_, _, deferfn := topframe.TopmostDefer.DeferredFunc(p)
-		var err error
-		deferpc, err = FirstPCAfterPrologue(p, deferfn, false)
-		if err != nil {
-			return 0, err
+		if deferfn != nil {
+			var err error
+			deferpc, err = FirstPCAfterPrologue(p, deferfn, false)
+			if err != nil {
+				return 0, err
+			}
 		}
 	}
 	if deferpc != 0 && deferpc != topframe.Current.PC {

pkg/terminal/command.go

@@ -286,7 +286,11 @@ Groups goroutines by the value of the label with the specified key.
 Called without arguments it will show information about the current goroutine.
 Called with a single argument it will switch to the specified goroutine.
 Called with more arguments it will execute a command on the specified goroutine.`},
-		{aliases: []string{"breakpoints", "bp"}, group: breakCmds, cmdFn: breakpoints, helpMsg: "Print out info for active breakpoints."},
+		{aliases: []string{"breakpoints", "bp"}, group: breakCmds, cmdFn: breakpoints, helpMsg: `Print out info for active breakpoints.
+	
+	breakpoints [-a]
+
+Specifying -a prints all physical breakpoint, including internal breakpoints.`},
 		{aliases: []string{"print", "p"}, group: dataCmds, allowedPrefixes: onPrefix | deferredPrefix, cmdFn: printVar, helpMsg: `Evaluate an expression.
 
 	[goroutine <n>] [frame <m>] print [%format] <expression>
@@ -1674,7 +1678,7 @@ func clear(t *Term, ctx callContext, args string) error {
 }
 
 func clearAll(t *Term, ctx callContext, args string) error {
-	breakPoints, err := t.client.ListBreakpoints()
+	breakPoints, err := t.client.ListBreakpoints(false)
 	if err != nil {
 		return err
 	}
@@ -1740,13 +1744,17 @@ func (a byID) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a byID) Less(i, j int) bool { return a[i].ID < a[j].ID }
 
 func breakpoints(t *Term, ctx callContext, args string) error {
-	breakPoints, err := t.client.ListBreakpoints()
+	breakPoints, err := t.client.ListBreakpoints(args == "-a")
 	if err != nil {
 		return err
 	}
 	sort.Sort(byID(breakPoints))
 	for _, bp := range breakPoints {
-		fmt.Printf("%s at %v (%d)\n", formatBreakpointName(bp, true), t.formatBreakpointLocation(bp), bp.TotalHitCount)
+		enabled := "(enabled)"
+		if bp.Disabled {
+			enabled = "(disabled)"
+		}
+		fmt.Printf("%s %s at %v (%d)\n", formatBreakpointName(bp, true), enabled, t.formatBreakpointLocation(bp), bp.TotalHitCount)
 
 		attrs := formatBreakpointAttrs("\t", bp, false)
 
@@ -1791,6 +1799,9 @@ func formatBreakpointAttrs(prefix string, bp *api.Breakpoint, includeTrace bool)
 	if includeTrace && bp.Tracepoint {
 		attrs = append(attrs, fmt.Sprintf("%strace", prefix))
 	}
+	for i := range bp.VerboseDescr {
+		attrs = append(attrs, fmt.Sprintf("%s%s", prefix, bp.VerboseDescr[i]))
+	}
 	return attrs
 }
 
@@ -2602,6 +2613,10 @@ func printcontext(t *Term, state *api.DebuggerState) {
 	if state.When != "" {
 		fmt.Println(state.When)
 	}
+
+	for _, watchpoint := range state.WatchOutOfScope {
+		fmt.Printf("%s went out of scope and was cleared\n", formatBreakpointName(watchpoint, true))
+	}
 }
 
 func printcontextLocation(t *Term, loc api.Location) {
@@ -3113,11 +3128,7 @@ func formatBreakpointName(bp *api.Breakpoint, upcase bool) string {
 	if bp.WatchExpr != "" && bp.WatchExpr != bp.Name {
 		return fmt.Sprintf("%s %s on [%s]", thing, id, bp.WatchExpr)
 	}
-	state := "(enabled)"
-	if bp.Disabled {
-		state = "(disabled)"
-	}
-	return fmt.Sprintf("%s %s %s", thing, id, state)
+	return fmt.Sprintf("%s %s", thing, id)
 }
 
 func (t *Term) formatBreakpointLocation(bp *api.Breakpoint) string {

pkg/terminal/starbind/starlark_mapping.go

@@ -834,6 +834,24 @@ func (env *Env) starlarkPredeclare() starlark.StringDict {
 		}
 		var rpcArgs rpc2.ListBreakpointsIn
 		var rpcRet rpc2.ListBreakpointsOut
+		if len(args) > 0 && args[0] != starlark.None {
+			err := unmarshalStarlarkValue(args[0], &rpcArgs.All, "All")
+			if err != nil {
+				return starlark.None, decorateError(thread, err)
+			}
+		}
+		for _, kv := range kwargs {
+			var err error
+			switch kv[0].(starlark.String) {
+			case "All":
+				err = unmarshalStarlarkValue(kv[1], &rpcArgs.All, "All")
+			default:
+				err = fmt.Errorf("unknown argument %q", kv[0])
+			}
+			if err != nil {
+				return starlark.None, decorateError(thread, err)
+			}
+		}
 		err := env.ctx.Client().CallAPI("ListBreakpoints", &rpcArgs, &rpcRet)
 		if err != nil {
 			return starlark.None, err

service/api/types.go

@@ -39,6 +39,9 @@ type DebuggerState struct {
 	// While NextInProgress is set further requests for next or step may be rejected.
 	// Either execute continue until NextInProgress is false or call CancelNext
 	NextInProgress bool
+	// WatchOutOfScope contains the list of watchpoints that went out of scope
+	// during the last continue.
+	WatchOutOfScope []*Breakpoint
 	// Exited indicates whether the debugged process has exited.
 	Exited     bool `json:"exited"`
 	ExitStatus int  `json:"exitStatus"`
@@ -108,6 +111,8 @@ type Breakpoint struct {
 	WatchExpr string
 	WatchType WatchType
 
+	VerboseDescr []string `json:"VerboseDescr,omitempty"`
+
 	// number of times a breakpoint has been reached in a certain goroutine
 	HitCount map[string]uint64 `json:"hitCount"`
 	// number of times a breakpoint has been reached

service/client.go

@@ -69,7 +69,7 @@ type Client interface {
 	// CreateWatchpoint creates a new watchpoint.
 	CreateWatchpoint(api.EvalScope, string, api.WatchType) (*api.Breakpoint, error)
 	// ListBreakpoints gets all breakpoints.
-	ListBreakpoints() ([]*api.Breakpoint, error)
+	ListBreakpoints(bool) ([]*api.Breakpoint, error)
 	// ClearBreakpoint deletes a breakpoint by ID.
 	ClearBreakpoint(id int) (*api.Breakpoint, error)
 	// ClearBreakpointByName deletes a breakpoint by name

service/dap/server.go

@@ -1367,7 +1367,7 @@ func (s *Server) clearBreakpoints(existingBps map[string]*api.Breakpoint, bpAdde
 }
 
 func (s *Server) getMatchingBreakpoints(prefix string) map[string]*api.Breakpoint {
-	existing := s.debugger.Breakpoints()
+	existing := s.debugger.Breakpoints(false)
 	matchingBps := make(map[string]*api.Breakpoint, len(existing))
 	for _, bp := range existing {
 		// Skip special breakpoints such as for panic.

service/debugger/debugger.go

@@ -623,6 +623,11 @@ func (d *Debugger) state(retLoadCfg *proc.LoadConfig) (*api.DebuggerState, error
 		state.When, _ = d.target.When()
 	}
 
+	state.WatchOutOfScope = make([]*api.Breakpoint, 0, len(d.target.Breakpoints().WatchOutOfScope))
+	for _, bp := range d.target.Breakpoints().WatchOutOfScope {
+		state.WatchOutOfScope = append(state.WatchOutOfScope, api.ConvertBreakpoint(bp))
+	}
+
 	return state, nil
 }
 
@@ -946,11 +951,21 @@ func (d *Debugger) clearBreakpoint(requestedBp *api.Breakpoint) (*api.Breakpoint
 }
 
 // Breakpoints returns the list of current breakpoints.
-func (d *Debugger) Breakpoints() []*api.Breakpoint {
+func (d *Debugger) Breakpoints(all bool) []*api.Breakpoint {
 	d.targetMutex.Lock()
 	defer d.targetMutex.Unlock()
 
-	bps := api.ConvertBreakpoints(d.breakpoints())
+	var bps []*api.Breakpoint
+
+	if !all {
+		bps = api.ConvertBreakpoints(d.breakpoints())
+	} else {
+		for _, bp := range d.target.Breakpoints().M {
+			abp := api.ConvertBreakpoint(bp)
+			abp.VerboseDescr = bp.VerboseDescr()
+			bps = append(bps, abp)
+		}
+	}
 
 	for _, bp := range d.disabledBreakpoints {
 		bps = append(bps, bp)

service/rpc1/server.go

@@ -98,7 +98,7 @@ func (s *RPCServer) StacktraceGoroutine(args *StacktraceGoroutineArgs, locations
 }
 
 func (s *RPCServer) ListBreakpoints(arg interface{}, breakpoints *[]*api.Breakpoint) error {
-	*breakpoints = s.debugger.Breakpoints()
+	*breakpoints = s.debugger.Breakpoints(false)
 	return nil
 }
 

service/rpc2/client.go

@@ -253,9 +253,9 @@ func (c *RPCClient) CreateWatchpoint(scope api.EvalScope, expr string, wtype api
 	return out.Breakpoint, err
 }
 
-func (c *RPCClient) ListBreakpoints() ([]*api.Breakpoint, error) {
+func (c *RPCClient) ListBreakpoints(all bool) ([]*api.Breakpoint, error) {
 	var out ListBreakpointsOut
-	err := c.call("ListBreakpoints", ListBreakpointsIn{}, &out)
+	err := c.call("ListBreakpoints", ListBreakpointsIn{all}, &out)
 	return out.Breakpoints, err
 }
 

service/rpc2/server.go

@@ -227,6 +227,7 @@ func (s *RPCServer) Ancestors(arg AncestorsIn, out *AncestorsOut) error {
 }
 
 type ListBreakpointsIn struct {
+	All bool
 }
 
 type ListBreakpointsOut struct {
@@ -235,7 +236,7 @@ type ListBreakpointsOut struct {
 
 // ListBreakpoints gets all breakpoints.
 func (s *RPCServer) ListBreakpoints(arg ListBreakpointsIn, out *ListBreakpointsOut) error {
-	out.Breakpoints = s.debugger.Breakpoints()
+	out.Breakpoints = s.debugger.Breakpoints(arg.All)
 	return nil
 }
 

service/test/common_test.go

@@ -9,6 +9,8 @@ import (
 	"testing"
 
 	"github.com/go-delve/delve/service/api"
+	"github.com/go-delve/delve/service/rpc1"
+	"github.com/go-delve/delve/service/rpc2"
 )
 
 func assertNoError(err error, t *testing.T, s string) {
@@ -63,8 +65,15 @@ type BreakpointLister interface {
 	ListBreakpoints() ([]*api.Breakpoint, error)
 }
 
-func countBreakpoints(t *testing.T, c BreakpointLister) int {
-	bps, err := c.ListBreakpoints()
+func countBreakpoints(t *testing.T, c interface{}) int {
+	var bps []*api.Breakpoint
+	var err error
+	switch c := c.(type) {
+	case *rpc2.RPCClient:
+		bps, err = c.ListBreakpoints(false)
+	case *rpc1.RPCClient:
+		bps, err = c.ListBreakpoints()
+	}
 	assertNoError(err, t, "ListBreakpoints()")
 	bpcount := 0
 	for _, bp := range bps {

service/test/integration2_test.go

@@ -196,7 +196,7 @@ func TestRestart_duringStop(t *testing.T) {
 		if c.ProcessPid() == origPid {
 			t.Fatal("did not spawn new process, has same PID")
 		}
-		bps, err := c.ListBreakpoints()
+		bps, err := c.ListBreakpoints(false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -1631,7 +1631,7 @@ func TestClientServer_RestartBreakpointPosition(t *testing.T) {
 		assertNoError(err, t, "Halt")
 		_, err = c.Restart(false)
 		assertNoError(err, t, "Restart")
-		bps, err := c.ListBreakpoints()
+		bps, err := c.ListBreakpoints(false)
 		assertNoError(err, t, "ListBreakpoints")
 		for _, bp := range bps {
 			if bp.Name == bpBefore.Name {
@@ -2138,7 +2138,7 @@ func TestDoubleCreateBreakpoint(t *testing.T) {
 		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.main", Line: 1, Name: "firstbreakpoint", Tracepoint: true})
 		assertNoError(err, t, "CreateBreakpoint 1")
 
-		bps, err := c.ListBreakpoints()
+		bps, err := c.ListBreakpoints(false)
 		assertNoError(err, t, "ListBreakpoints 1")
 
 		t.Logf("breakpoints before second call:")
@@ -2151,7 +2151,7 @@ func TestDoubleCreateBreakpoint(t *testing.T) {
 		_, err = c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.main", Line: 1, Name: "secondbreakpoint", Tracepoint: true})
 		assertError(err, t, "CreateBreakpoint 2") // breakpoint exists
 
-		bps, err = c.ListBreakpoints()
+		bps, err = c.ListBreakpoints(false)
 		assertNoError(err, t, "ListBreakpoints 2")
 
 		t.Logf("breakpoints after second call:")
@@ -2201,7 +2201,7 @@ func TestClearLogicalBreakpoint(t *testing.T) {
 		}
 		_, err = c.ClearBreakpoint(bp.ID)
 		assertNoError(err, t, "ClearBreakpoint()")
-		bps, err := c.ListBreakpoints()
+		bps, err := c.ListBreakpoints(false)
 		assertNoError(err, t, "ListBreakpoints()")
 		for _, curbp := range bps {
 			if curbp.ID == bp.ID {
@@ -2322,7 +2322,7 @@ func TestDetachLeaveRunning(t *testing.T) {
 
 func assertNoDuplicateBreakpoints(t *testing.T, c service.Client) {
 	t.Helper()
-	bps, _ := c.ListBreakpoints()
+	bps, _ := c.ListBreakpoints(false)
 	seen := make(map[int]bool)
 	for _, bp := range bps {
 		t.Logf("%#v\n", bp)