Go 1.17 support branch (#2451)

* proc: support new Go 1.17 panic/defer mechanism

Go 1.17 will create wrappers for deferred calls that take arguments.
Change defer reading code so that wrappers are automatically unwrapped.

Also the deferred function is called directly by runtime.gopanic, without going through runtime.callN which means that sometimes when a panic happens the stack is either:

0. deferred function call
1. deferred call wrapper
2. runtime.gopanic

or:

0. deferred function call
1. runtime.gopanic

instead of always being:

0. deferred function call
1. runtime.callN
2. runtime.gopanic

the isPanicCall check is changed accordingly.

* test: miscellaneous minor test fixes for Go 1.17

* proc: resolve inlined calls when stepping out of runtime.breakpoint

Calls to runtime.Breakpoint are inlined in Go 1.17 when inlining is
enabled, resolve inlined calls in stepInstructionOut.

* proc: add support for debugCallV2 with regabi

This change adds support for the new debug call protocol which had to
change for the new register ABI introduced in Go 1.17.

Summary of changes:
- Abstracts over the debug call version depending on the Go version
  found in the binary.
- Uses R12 instead of RAX as the debug protocol register when the binary
  is from Go 1.17 or later.
- Creates a variable directly from the DWARF entry for function
  arguments to support passing arguments however the ABI expects.
- Computes a very conservative stack frame size for the call when
  injecting a call into a Go process whose version is >=1.17.

Co-authored-by: Michael Anthony Knyszek <mknyszek@google.com>
Co-authored-by: Alessandro Arzilli <alessandro.arzilli@gmail.com>

* TeamCity: enable tests on go-tip

* goversion: version compatibility bump

* TeamCity: fix go-tip builds on macOS/arm64

Co-authored-by: Michael Anthony Knyszek <mknyszek@google.com>

_fixtures/fncall.go

@@ -168,6 +168,14 @@ func (_ X2) CallMe(i int) int {
 	return i * i
 }
 
+func regabistacktest(s1, s2, s3, s4, s5 string, n uint8) (string, string, string, string, string, uint8) {
+	return s1 + s2, s2 + s3, s3 + s4, s4 + s5, s5 + s1, 2 * n
+}
+
+func regabistacktest2(n1, n2, n3, n4, n5, n6, n7, n8, n9, n10 int) (int, int, int, int, int, int, int, int, int, int) {
+	return n1 + n2, n2 + n3, n3 + n4, n4 + n5, n5 + n6, n6 + n7, n7 + n8, n8 + n9, n9 + n10, n10 + n1
+}
+
 func main() {
 	one, two := 1, 2
 	intslice := []int{1, 2, 3}
@@ -200,5 +208,5 @@ func main() {
 	d.Method()
 	d.Base.Method()
 	x.CallMe()
-	fmt.Println(one, two, zero, call, call0, call2, callexit, callpanic, callbreak, callstacktrace, stringsJoin, intslice, stringslice, comma, a.VRcvr, a.PRcvr, pa, vable_a, vable_pa, pable_pa, fn2clos, fn2glob, fn2valmeth, fn2ptrmeth, fn2nil, ga, escapeArg, a2, square, intcallpanic, onetwothree, curriedAdd, getAStruct, getAStructPtr, getVRcvrableFromAStruct, getPRcvrableFromAStructPtr, getVRcvrableFromAStructPtr, pa2, noreturncall, str, d, x, x2.CallMe(5), longstrs)
+	fmt.Println(one, two, zero, call, call0, call2, callexit, callpanic, callbreak, callstacktrace, stringsJoin, intslice, stringslice, comma, a.VRcvr, a.PRcvr, pa, vable_a, vable_pa, pable_pa, fn2clos, fn2glob, fn2valmeth, fn2ptrmeth, fn2nil, ga, escapeArg, a2, square, intcallpanic, onetwothree, curriedAdd, getAStruct, getAStructPtr, getVRcvrableFromAStruct, getPRcvrableFromAStructPtr, getVRcvrableFromAStructPtr, pa2, noreturncall, str, d, x, x2.CallMe(5), longstrs, regabistacktest, regabistacktest2)
 }

_scripts/test_linux.sh

@@ -20,7 +20,7 @@ function getgo {
 }
 
 if [ "$version" = "gotip" ]; then
-	exit 0
+	#exit 0
 	echo Building Go from tip
 	getgo $(curl https://golang.org/VERSION?m=text)
 	export GOROOT_BOOTSTRAP=$GOROOT

_scripts/test_mac.sh

@@ -8,10 +8,17 @@ ARCH=$2
 TMPDIR=$3
 
 if [ "$GOVERSION" = "gotip" ]; then
-    exit 0
+    #exit 0
     bootstrapver=$(curl https://golang.org/VERSION?m=text)
+    cd $TMPDIR
     curl -sSL "https://storage.googleapis.com/golang/$bootstrapver.darwin-$ARCH.tar.gz" | tar -xz
+    cd -
+    if [ -x $TMPDIR/go-tip ]; then
+    	cd $TMPDIR/go-tip
+    	git pull origin
+    else
     	git clone https://go.googlesource.com/go $TMPDIR/go-tip
+    fi
     export GOROOT_BOOTSTRAP=$TMPDIR/go
     export GOROOT=$TMPDIR/go-tip
     cd $TMPDIR/go-tip/src

_scripts/test_windows.ps1

@@ -32,7 +32,7 @@ function GetGo($version) {
 }
 
 if ($version -eq "gotip") {
-    Exit 0
+    #Exit 0
     $latest = Invoke-WebRequest -Uri https://golang.org/VERSION?m=text -UseBasicParsing | Select-Object -ExpandProperty Content
     GetGo $latest
     $env:GOROOT_BOOTSTRAP = $env:GOROOT

pkg/goversion/compat.go

@@ -8,7 +8,7 @@ var (
 	MinSupportedVersionOfGoMajor = 1
 	MinSupportedVersionOfGoMinor = 14
 	MaxSupportedVersionOfGoMajor = 1
-	MaxSupportedVersionOfGoMinor = 16
+	MaxSupportedVersionOfGoMinor = 17
 	goTooOldErr                  = fmt.Errorf("Version of Go is too old for this version of Delve (minimum supported version %d.%d, suppress this error with --check-go-version=false)", MinSupportedVersionOfGoMajor, MinSupportedVersionOfGoMinor)
 	dlvTooOldErr                 = fmt.Errorf("Version of Delve is too old for this version of Go (maximum supported version %d.%d, suppress this error with --check-go-version=false)", MaxSupportedVersionOfGoMajor, MaxSupportedVersionOfGoMinor)
 )

pkg/proc/bininfo.go

@@ -100,6 +100,12 @@ type BinaryInfo struct {
 	// function starts.
 	inlinedCallLines map[fileLine][]uint64
 
+	// dwrapUnwrapCache caches unwrapping of defer wrapper functions (dwrap)
+	dwrapUnwrapCache map[uint64]*Function
+
+	// Go 1.17 register ABI is enabled.
+	regabi bool
+
 	logger *logrus.Entry
 }
 
@@ -657,6 +663,7 @@ type Image struct {
 	compileUnits []*compileUnit // compileUnits is sorted by increasing DWARF offset
 
 	dwarfTreeCache      *simplelru.LRU
+	runtimeMallocgcTree *godwarf.Tree // patched version of runtime.mallocgc's DIE
 
 	// runtimeTypeToDIE maps between the offset of a runtime._type in
 	// runtime.moduledata.types and the offset of the DIE in debug_info. This
@@ -782,6 +789,9 @@ func (image *Image) LoadError() error {
 }
 
 func (image *Image) getDwarfTree(off dwarf.Offset) (*godwarf.Tree, error) {
+	if image.runtimeMallocgcTree != nil && off == image.runtimeMallocgcTree.Offset {
+		return image.runtimeMallocgcTree, nil
+	}
 	if r, ok := image.dwarfTreeCache.Get(off); ok {
 		return r.(*godwarf.Tree), nil
 	}
@@ -1681,6 +1691,9 @@ func (bi *BinaryInfo) loadDebugInfoMaps(image *Image, debugInfoBytes, debugLineB
 	if bi.inlinedCallLines == nil {
 		bi.inlinedCallLines = make(map[fileLine][]uint64)
 	}
+	if bi.dwrapUnwrapCache == nil {
+		bi.dwrapUnwrapCache = make(map[uint64]*Function)
+	}
 
 	image.runtimeTypeToDIE = make(map[uint64]runtimeTypeDIE)
 
@@ -1735,6 +1748,13 @@ func (bi *BinaryInfo) loadDebugInfoMaps(image *Image, debugInfoBytes, debugLineB
 					cu.optimized = goversion.ProducerAfterOrEqual(cu.producer, 1, 10)
 				} else {
 					cu.optimized = !strings.Contains(cu.producer[semicolon:], "-N") || !strings.Contains(cu.producer[semicolon:], "-l")
+					const regabi = " regabi"
+					if i := strings.Index(cu.producer[semicolon:], regabi); i > 0 {
+						i += semicolon
+						if i+len(regabi) >= len(cu.producer) || cu.producer[i+len(regabi)] == ' ' {
+							bi.regabi = true
+						}
+					}
 					cu.producer = cu.producer[:semicolon]
 				}
 			}
@@ -1775,6 +1795,22 @@ func (bi *BinaryInfo) loadDebugInfoMaps(image *Image, debugInfoBytes, debugLineB
 	sort.Strings(bi.Sources)
 	bi.Sources = uniq(bi.Sources)
 
+	if bi.regabi {
+		// prepare patch for runtime.mallocgc's DIE
+		fn := bi.LookupFunc["runtime.mallocgc"]
+		if fn != nil {
+			tree, err := image.getDwarfTree(fn.offset)
+			if err == nil {
+				tree.Children, err = regabiMallocgcWorkaround(bi)
+				if err != nil {
+					bi.logger.Errorf("could not patch runtime.mallogc: %v", err)
+				} else {
+					image.runtimeMallocgcTree = tree
+				}
+			}
+		}
+	}
+
 	if cont != nil {
 		cont()
 	}

pkg/proc/breakpoints.go

@@ -194,7 +194,7 @@ func (bpstate *BreakpointState) checkCond(thread Thread) {
 		var err error
 		frames, err := ThreadStacktrace(thread, 2)
 		if err == nil {
-			nextDeferOk = isPanicCall(frames)
+			nextDeferOk, _ = isPanicCall(frames)
 			if !nextDeferOk {
 				nextDeferOk, _ = isDeferReturnCall(frames, bpstate.DeferReturns)
 			}
@@ -239,8 +239,25 @@ func (bpstate *BreakpointState) checkHitCond(thread Thread) {
 	}
 }
 
-func isPanicCall(frames []Stackframe) bool {
+func isPanicCall(frames []Stackframe) (bool, int) {
-	return len(frames) >= 3 && frames[2].Current.Fn != nil && frames[2].Current.Fn.Name == "runtime.gopanic"
+	// In Go prior to 1.17 the call stack for a panic is:
+	//  0. deferred function call
+	//  1. runtime.callN
+	//  2. runtime.gopanic
+	// in Go after 1.17 it is either:
+	//  0. deferred function call
+	//  1. deferred call wrapper
+	//  2. runtime.gopanic
+	// or:
+	//  0. deferred function call
+	//  1. runtime.gopanic
+	if len(frames) >= 3 && frames[2].Current.Fn != nil && frames[2].Current.Fn.Name == "runtime.gopanic" {
+		return true, 2
+	}
+	if len(frames) >= 2 && frames[1].Current.Fn != nil && frames[1].Current.Fn.Name == "runtime.gopanic" {
+		return true, 1
+	}
+	return false, 0
 }
 
 func isDeferReturnCall(frames []Stackframe, deferReturns []uint64) (bool, uint64) {

pkg/proc/fncall.go

@@ -42,8 +42,12 @@ import (
 const (
 	debugCallFunctionNamePrefix1 = "debugCall"
 	debugCallFunctionNamePrefix2 = "runtime.debugCall"
-	debugCallFunctionName        = "runtime.debugCallV1"
+	maxDebugCallVersion          = 2
 	maxArgFrameSize              = 65535
+
+	// maxRegArgBytes is extra padding for ABI1 call injections, equivalent to
+	// the maximum space occupied by register arguments.
+	maxRegArgBytes = 9*8 + 15*8 // TODO: Make this generic for other platforms.
 )
 
 var (
@@ -163,7 +167,7 @@ func EvalExpressionWithCalls(t *Target, g *G, expr string, retLoadCfg LoadConfig
 		return errFuncCallInProgress
 	}
 
-	dbgcallfn := bi.LookupFunc[debugCallFunctionName]
+	dbgcallfn, _ := debugCallFunction(bi)
 	if dbgcallfn == nil {
 		return errFuncCallUnsupported
 	}
@@ -272,7 +276,7 @@ func evalFunctionCall(scope *EvalScope, node *ast.CallExpr) (*Variable, error) {
 		return nil, errFuncCallUnsupportedBackend
 	}
 
-	dbgcallfn := bi.LookupFunc[debugCallFunctionName]
+	dbgcallfn, dbgcallversion := debugCallFunction(bi)
 	if dbgcallfn == nil {
 		return nil, errFuncCallUnsupported
 	}
@@ -289,7 +293,11 @@ func evalFunctionCall(scope *EvalScope, node *ast.CallExpr) (*Variable, error) {
 	if regs.SP()-256 <= stacklo {
 		return nil, errNotEnoughStack
 	}
-	if bi.Arch.RegistersToDwarfRegisters(0, regs).Reg(regnum.AMD64_Rax) == nil { //TODO(aarzilli): make this generic when call injection is supported on other architectures
+	protocolReg, ok := debugCallProtocolReg(dbgcallversion)
+	if !ok {
+		return nil, errFuncCallUnsupported
+	}
+	if bi.Arch.RegistersToDwarfRegisters(0, regs).Reg(protocolReg) == nil {
 		return nil, errFuncCallUnsupportedBackend
 	}
 
@@ -347,7 +355,7 @@ func evalFunctionCall(scope *EvalScope, node *ast.CallExpr) (*Variable, error) {
 		scope.Regs.FrameBase = fboff + int64(scope.g.stack.hi)
 		scope.Regs.CFA = scope.frameOffset + int64(scope.g.stack.hi)
 
-		finished := funcCallStep(scope, &fncall, g.Thread)
+		finished := funcCallStep(scope, &fncall, g.Thread, protocolReg, dbgcallfn.Name)
 		if finished {
 			break
 		}
@@ -501,19 +509,20 @@ type funcCallArg struct {
 	name       string
 	typ        godwarf.Type
 	off        int64
+	dwarfEntry *godwarf.Tree // non-nil if Go 1.17+
 	isret      bool
 }
 
 // funcCallEvalArgs evaluates the arguments of the function call, copying
-// the into the argument frame starting at argFrameAddr.
+// them into the argument frame starting at argFrameAddr.
-func funcCallEvalArgs(scope *EvalScope, fncall *functionCallState, argFrameAddr uint64) error {
+func funcCallEvalArgs(scope *EvalScope, fncall *functionCallState, formalScope *EvalScope) error {
 	if scope.g == nil {
 		// this should never happen
 		return errNoGoroutine
 	}
 
 	if fncall.receiver != nil {
-		err := funcCallCopyOneArg(scope, fncall, fncall.receiver, &fncall.formalArgs[0], argFrameAddr)
+		err := funcCallCopyOneArg(scope, fncall, fncall.receiver, &fncall.formalArgs[0], formalScope)
 		if err != nil {
 			return err
 		}
@@ -529,7 +538,7 @@ func funcCallEvalArgs(scope *EvalScope, fncall *functionCallState, argFrameAddr
 		}
 		actualArg.Name = exprToString(fncall.expr.Args[i])
 
-		err = funcCallCopyOneArg(scope, fncall, actualArg, formalArg, argFrameAddr)
+		err = funcCallCopyOneArg(scope, fncall, actualArg, formalArg, formalScope)
 		if err != nil {
 			return err
 		}
@@ -538,7 +547,7 @@ func funcCallEvalArgs(scope *EvalScope, fncall *functionCallState, argFrameAddr
 	return nil
 }
 
-func funcCallCopyOneArg(scope *EvalScope, fncall *functionCallState, actualArg *Variable, formalArg *funcCallArg, argFrameAddr uint64) error {
+func funcCallCopyOneArg(scope *EvalScope, fncall *functionCallState, actualArg *Variable, formalArg *funcCallArg, formalScope *EvalScope) error {
 	if scope.callCtx.checkEscape {
 		//TODO(aarzilli): only apply the escapeCheck to leaking parameters.
 		if err := escapeCheck(actualArg, formalArg.name, scope.g.stack); err != nil {
@@ -554,7 +563,16 @@ func funcCallCopyOneArg(scope *EvalScope, fncall *functionCallState, actualArg *
 	//TODO(aarzilli): autmoatic wrapping in interfaces for cases not handled
 	// by convertToEface.
 
-	formalArgVar := newVariable(formalArg.name, uint64(formalArg.off+int64(argFrameAddr)), formalArg.typ, scope.BinInfo, scope.Mem)
+	var formalArgVar *Variable
+	if formalArg.dwarfEntry != nil {
+		var err error
+		formalArgVar, err = extractVarInfoFromEntry(scope.target, formalScope.BinInfo, formalScope.image(), formalScope.Regs, formalScope.Mem, formalArg.dwarfEntry)
+		if err != nil {
+			return err
+		}
+	} else {
+		formalArgVar = newVariable(formalArg.name, uint64(formalArg.off+int64(formalScope.Regs.CFA)), formalArg.typ, scope.BinInfo, scope.Mem)
+	}
 	if err := scope.setValue(formalArgVar, actualArg, actualArg.Name); err != nil {
 		return err
 	}
@@ -563,16 +581,26 @@ func funcCallCopyOneArg(scope *EvalScope, fncall *functionCallState, actualArg *
 }
 
 func funcCallArgs(fn *Function, bi *BinaryInfo, includeRet bool) (argFrameSize int64, formalArgs []funcCallArg, err error) {
-	const CFA = 0x1000
-
 	dwarfTree, err := fn.cu.image.getDwarfTree(fn.offset)
 	if err != nil {
 		return 0, nil, fmt.Errorf("DWARF read error: %v", err)
 	}
 
-	varEntries := reader.Variables(dwarfTree, fn.Entry, int(^uint(0)>>1), reader.VariablesSkipInlinedSubroutines)
+	producer := bi.Producer()
+	trustArgOrder := producer != "" && goversion.ProducerAfterOrEqual(bi.Producer(), 1, 12)
 
-	trustArgOrder := bi.Producer() != "" && goversion.ProducerAfterOrEqual(bi.Producer(), 1, 12)
+	if bi.regabi && fn.cu.optimized && fn.Name != "runtime.mallocgc" {
+		// Debug info for function arguments on optimized functions is currently
+		// too incomplete to attempt injecting calls to arbitrary optimized
+		// functions.
+		// Prior to regabi we could do this because the ABI was simple enough to
+		// manually encode it in Delve.
+		// Runtime.mallocgc is an exception, we specifically patch it's DIE to be
+		// correct for call injection purposes.
+		return 0, nil, fmt.Errorf("can not call optimized function %s when regabi is in use", fn.Name)
+	}
+
+	varEntries := reader.Variables(dwarfTree, fn.Entry, int(^uint(0)>>1), reader.VariablesSkipInlinedSubroutines)
 
 	// typechecks arguments, calculates argument frame size
 	for _, entry := range varEntries {
@@ -584,6 +612,48 @@ func funcCallArgs(fn *Function, bi *BinaryInfo, includeRet bool) (argFrameSize i
 			return 0, nil, err
 		}
 		typ = resolveTypedef(typ)
+
+		var formalArg *funcCallArg
+		if bi.regabi {
+			formalArg, err = funcCallArgRegABI(fn, bi, entry, argname, typ, &argFrameSize)
+		} else {
+			formalArg, err = funcCallArgOldABI(fn, bi, entry, argname, typ, trustArgOrder, &argFrameSize)
+		}
+		if err != nil {
+			return 0, nil, err
+		}
+		if !formalArg.isret || includeRet {
+			formalArgs = append(formalArgs, *formalArg)
+		}
+	}
+
+	if bi.regabi {
+		// The argument frame size is computed conservatively, assuming that
+		// there's space for each argument on the stack even if its passed in
+		// registers. Unfortunately this isn't quite enough because the register
+		// assignment algorithm Go uses can result in an amount of additional
+		// space used due to alignment requirements, bounded by the number of argument registers.
+		// Because we currently don't have an easy way to obtain the frame size,
+		// let's be even more conservative.
+		// A safe lower-bound on the size of the argument frame includes space for
+		// each argument plus the total bytes of register arguments.
+		// This is derived from worst-case alignment padding of up to
+		// (pointer-word-bytes - 1) per argument passed in registers.
+		// See: https://github.com/go-delve/delve/pull/2451#discussion_r665761531
+		// TODO: Make this generic for other platforms.
+		argFrameSize = alignAddr(argFrameSize, 8)
+		argFrameSize += maxRegArgBytes
+	}
+
+	sort.Slice(formalArgs, func(i, j int) bool {
+		return formalArgs[i].off < formalArgs[j].off
+	})
+
+	return argFrameSize, formalArgs, nil
+}
+
+func funcCallArgOldABI(fn *Function, bi *BinaryInfo, entry reader.Variable, argname string, typ godwarf.Type, trustArgOrder bool, pargFrameSize *int64) (*funcCallArg, error) {
+	const CFA = 0x1000
 	var off int64
 
 	locprog, _, err := bi.locationExpr(entry, dwarf.AttrLocation, fn.Entry)
@@ -602,7 +672,7 @@ func funcCallArgs(fn *Function, bi *BinaryInfo, includeRet bool) (argFrameSize i
 	}
 	if err != nil {
 		if !trustArgOrder {
-				return 0, nil, err
+			return nil, err
 		}
 
 		// With Go version 1.12 or later we can trust that the arguments appear
@@ -611,24 +681,25 @@ func funcCallArgs(fn *Function, bi *BinaryInfo, includeRet bool) (argFrameSize i
 		// With this we can call optimized functions (which sometimes do not have
 		// an argument address, due to a compiler bug) as well as runtime
 		// functions (which are always optimized).
-			off = argFrameSize
+		off = *pargFrameSize
 		off = alignAddr(off, typ.Align())
 	}
 
-		if e := off + typ.Size(); e > argFrameSize {
+	if e := off + typ.Size(); e > *pargFrameSize {
-			argFrameSize = e
+		*pargFrameSize = e
 	}
 
-		if isret, _ := entry.Val(dwarf.AttrVarParam).(bool); !isret || includeRet {
+	isret, _ := entry.Val(dwarf.AttrVarParam).(bool)
-			formalArgs = append(formalArgs, funcCallArg{name: argname, typ: typ, off: off, isret: isret})
+	return &funcCallArg{name: argname, typ: typ, off: off, isret: isret}, nil
-		}
+}
-	}
 
-	sort.Slice(formalArgs, func(i, j int) bool {
+func funcCallArgRegABI(fn *Function, bi *BinaryInfo, entry reader.Variable, argname string, typ godwarf.Type, pargFrameSize *int64) (*funcCallArg, error) {
-		return formalArgs[i].off < formalArgs[j].off
+	// Conservatively calculate the full stack argument space for ABI0.
-	})
+	*pargFrameSize = alignAddr(*pargFrameSize, typ.Align())
+	*pargFrameSize += typ.Size()
 
-	return argFrameSize, formalArgs, nil
+	isret, _ := entry.Val(dwarf.AttrVarParam).(bool)
+	return &funcCallArg{name: argname, typ: typ, dwarfEntry: entry.Tree, isret: isret}, nil
 }
 
 // alignAddr rounds up addr to a multiple of align. Align must be a power of 2.
@@ -686,15 +757,15 @@ func escapeCheckPointer(addr uint64, name string, stack stack) error {
 }
 
 const (
-	debugCallAXPrecheckFailed   = 8
+	debugCallRegPrecheckFailed   = 8
-	debugCallAXCompleteCall     = 0
+	debugCallRegCompleteCall     = 0
-	debugCallAXReadReturn       = 1
+	debugCallRegReadReturn       = 1
-	debugCallAXReadPanic        = 2
+	debugCallRegReadPanic        = 2
-	debugCallAXRestoreRegisters = 16
+	debugCallRegRestoreRegisters = 16
 )
 
 // funcCallStep executes one step of the function call injection protocol.
-func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread) bool {
+func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread, protocolReg uint64, debugCallName string) bool {
 	p := callScope.callCtx.p
 	bi := p.BinInfo()
 
@@ -704,7 +775,7 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 		return true
 	}
 
-	rax := bi.Arch.RegistersToDwarfRegisters(0, regs).Uint64Val(regnum.AMD64_Rax) //TODO(aarzilli): make this generic when call injection is supported on other architectures
+	regval := bi.Arch.RegistersToDwarfRegisters(0, regs).Uint64Val(protocolReg)
 
 	if logflags.FnCall() {
 		loc, _ := thread.Location()
@@ -716,11 +787,11 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 				fnname = loc.Fn.Name
 			}
 		}
-		fncallLog("function call interrupt gid=%d (original) thread=%d rax=%#x (PC=%#x in %s)", callScope.g.ID, thread.ThreadID(), rax, pc, fnname)
+		fncallLog("function call interrupt gid=%d (original) thread=%d regval=%#x (PC=%#x in %s)", callScope.g.ID, thread.ThreadID(), regval, pc, fnname)
 	}
 
-	switch rax {
+	switch regval {
-	case debugCallAXPrecheckFailed:
+	case debugCallRegPrecheckFailed:
 		// get error from top of the stack and return it to user
 		errvar, err := readTopstackVariable(p, thread, regs, "string", loadFullValue)
 		if err != nil {
@@ -730,7 +801,7 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 		errvar.Name = "err"
 		fncall.err = fmt.Errorf("%v", constant.StringVal(errvar.Value))
 
-	case debugCallAXCompleteCall:
+	case debugCallRegCompleteCall:
 		p.fncallForG[callScope.g.ID].startThreadID = 0
 		// evaluate arguments of the target function, copy them into its argument frame and call the function
 		if fncall.fn == nil || fncall.receiver != nil || fncall.closureAddr != 0 {
@@ -763,8 +834,15 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 		cfa := regs.SP()
 		oldpc := regs.PC()
 		callOP(bi, thread, regs, fncall.fn.Entry)
+		formalScope, err := GoroutineScope(callScope.target, thread)
+		if formalScope != nil && formalScope.Regs.CFA != int64(cfa) {
+			// This should never happen, checking just to avoid hard to figure out disasters.
+			err = fmt.Errorf("mismatch in CFA %#x (calculated) %#x (expected)", formalScope.Regs.CFA, int64(cfa))
+		}
+		if err == nil {
+			err = funcCallEvalArgs(callScope, fncall, formalScope)
+		}
 
-		err := funcCallEvalArgs(callScope, fncall, cfa)
 		if err != nil {
 			// rolling back the call, note: this works because we called regs.Copy() above
 			setSP(thread, cfa)
@@ -774,7 +852,7 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 			break
 		}
 
-	case debugCallAXRestoreRegisters:
+	case debugCallRegRestoreRegisters:
 		// runtime requests that we restore the registers (all except pc and sp),
 		// this is also the last step of the function call protocol.
 		pc, sp := regs.PC(), regs.SP()
@@ -787,12 +865,12 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 		if err := setSP(thread, sp); err != nil {
 			fncall.err = fmt.Errorf("could not restore SP: %v", err)
 		}
-		if err := stepInstructionOut(p, thread, debugCallFunctionName, debugCallFunctionName); err != nil {
+		if err := stepInstructionOut(p, thread, debugCallName, debugCallName); err != nil {
-			fncall.err = fmt.Errorf("could not step out of %s: %v", debugCallFunctionName, err)
+			fncall.err = fmt.Errorf("could not step out of %s: %v", debugCallName, err)
 		}
 		return true
 
-	case debugCallAXReadReturn:
+	case debugCallRegReadReturn:
 		// read return arguments from stack
 		if fncall.panicvar != nil || fncall.lateCallFailure {
 			break
@@ -805,7 +883,7 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 
 		// pretend we are still inside the function we called
 		fakeFunctionEntryScope(retScope, fncall.fn, int64(regs.SP()), regs.SP()-uint64(bi.Arch.PtrSize()))
-		retScope.trustArgOrder = true
+		retScope.trustArgOrder = !bi.regabi
 
 		fncall.retvars, err = retScope.Locals()
 		if err != nil {
@@ -828,7 +906,7 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 			callScope.callCtx.stacks = append(callScope.callCtx.stacks, threadg.stack)
 		}
 
-	case debugCallAXReadPanic:
+	case debugCallRegReadPanic:
 		// read panic value from stack
 		fncall.panicvar, err = readTopstackVariable(p, thread, regs, "interface {}", callScope.callCtx.retLoadCfg)
 		if err != nil {
@@ -838,9 +916,9 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 		fncall.panicvar.Name = "~panic"
 
 	default:
-		// Got an unknown AX value, this is probably bad but the safest thing
+		// Got an unknown protocol register value, this is probably bad but the safest thing
 		// possible is to ignore it and hope it didn't matter.
-		fncallLog("unknown value of AX %#x", rax)
+		fncallLog("unknown value of protocol register %#x", regval)
 	}
 
 	return false
@@ -877,6 +955,7 @@ func fakeFunctionEntryScope(scope *EvalScope, fn *Function, cfa int64, sp uint64
 
 	scope.Regs.CFA = cfa
 	scope.Regs.Reg(scope.Regs.SPRegNum).Uint64Val = sp
+	scope.Regs.Reg(scope.Regs.PCRegNum).Uint64Val = fn.Entry
 
 	fn.cu.image.dwarfReader.Seek(fn.offset)
 	e, err := fn.cu.image.dwarfReader.Next()
@@ -1016,3 +1095,83 @@ func findCallInjectionStateForThread(t *Target, thread Thread) (*G, *callInjecti
 
 	return nil, nil, notfound()
 }
+
+// debugCallFunction searches for the debug call function in the binary and
+// uses this search to detect the debug call version.
+// Returns the debug call function and its version as an integer (the lowest
+// valid version is 1) or nil and zero.
+func debugCallFunction(bi *BinaryInfo) (*Function, int) {
+	for version := maxDebugCallVersion; version >= 1; version-- {
+		name := debugCallFunctionNamePrefix2 + "V" + strconv.Itoa(version)
+		fn, ok := bi.LookupFunc[name]
+		if ok && fn != nil {
+			return fn, version
+		}
+	}
+	return nil, 0
+}
+
+// debugCallProtocolReg returns the register ID (as defined in pkg/dwarf/regnum)
+// of the register used in the debug call protocol, given the debug call version.
+// Also returns a bool indicating whether the version is supported.
+func debugCallProtocolReg(version int) (uint64, bool) {
+	// TODO(aarzilli): make this generic when call injection is supported on other architectures.
+	var protocolReg uint64
+	switch version {
+	case 1:
+		protocolReg = regnum.AMD64_Rax
+	case 2:
+		protocolReg = regnum.AMD64_R12
+	default:
+		return 0, false
+	}
+	return protocolReg, true
+}
+
+type fakeEntry map[dwarf.Attr]interface{}
+
+func (e fakeEntry) Val(attr dwarf.Attr) interface{} {
+	return e[attr]
+}
+
+func regabiMallocgcWorkaround(bi *BinaryInfo) ([]*godwarf.Tree, error) {
+	var err1 error
+
+	t := func(name string) godwarf.Type {
+		if err1 != nil {
+			return nil
+		}
+		typ, err := bi.findType(name)
+		if err != nil {
+			err1 = err
+			return nil
+		}
+		return typ
+	}
+
+	m := func(name string, typ godwarf.Type, reg int, isret bool) *godwarf.Tree {
+		if err1 != nil {
+			return nil
+		}
+		var e fakeEntry = map[dwarf.Attr]interface{}{
+			dwarf.AttrName:     name,
+			dwarf.AttrType:     typ.Common().Offset,
+			dwarf.AttrLocation: []byte{byte(op.DW_OP_reg0) + byte(reg)},
+			dwarf.AttrVarParam: isret,
+		}
+
+		return &godwarf.Tree{
+			Entry: e,
+			Tag:   dwarf.TagFormalParameter,
+		}
+	}
+
+	r := []*godwarf.Tree{
+		m("size", t("uintptr"), regnum.AMD64_Rax, false),
+		m("typ", t("*runtime._type"), regnum.AMD64_Rbx, false),
+		m("needzero", t("bool"), regnum.AMD64_Rcx, false),
+		m("~r1", t("unsafe.Pointer"), regnum.AMD64_Rax, true),
+	}
+
+	return r, err1
+}

pkg/proc/proc_general_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"unsafe"
 
+	"github.com/go-delve/delve/pkg/goversion"
 	protest "github.com/go-delve/delve/pkg/proc/test"
 )
 
@@ -118,3 +119,16 @@ func TestDwarfVersion(t *testing.T) {
 		}
 	}
 }
+
+func TestRegabiFlagSentinel(t *testing.T) {
+	// Detect if the regabi flag in the producer string gets removed
+	if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 17) || runtime.GOARCH != "amd64" {
+		t.Skip("irrelevant before Go 1.17 or on non-amd64 architectures")
+	}
+	fixture := protest.BuildFixture("math", 0)
+	bi := NewBinaryInfo(runtime.GOOS, runtime.GOARCH)
+	assertNoError(bi.LoadBinaryInfo(fixture.Path, 0, nil), t, "LoadBinaryInfo")
+	if !bi.regabi {
+		t.Errorf("regabi flag not set")
+	}
+}

pkg/proc/proc_test.go

@@ -423,7 +423,7 @@ func testseq(program string, contFunc contFunc, testcases []nextTest, initialLoc
 	testseq2(t, program, initialLocation, seqTestcases)
 }
 
-const traceTestseq2 = false
+const traceTestseq2 = true
 
 func testseq2(t *testing.T, program string, initialLocation string, testcases []seqTest) {
 	testseq2Args(".", []string{}, 0, t, program, initialLocation, testcases)
@@ -1284,7 +1284,7 @@ func TestFrameEvaluation(t *testing.T) {
 				continue
 			}
 			t.Logf("Goroutine %d %#v", g.ID, g.Thread)
-			logStacktrace(t, p.BinInfo(), frames)
+			logStacktrace(t, p, frames)
 			for i := range frames {
 				if frames[i].Call.Fn != nil && frames[i].Call.Fn.Name == "main.agoroutine" {
 					frame = i
@@ -2245,7 +2245,7 @@ func TestStepCall(t *testing.T) {
 func TestStepCallPtr(t *testing.T) {
 	// Tests that Step works correctly when calling functions with a
 	// function pointer.
-	if goversion.VersionAfterOrEqual(runtime.Version(), 1, 11) {
+	if goversion.VersionAfterOrEqual(runtime.Version(), 1, 11) && !(goversion.VersionAfterOrEqual(runtime.Version(), 1, 17) && (runtime.GOARCH == "amd64")) {
 		testseq("teststepprog", contStep, []nextTest{
 			{9, 10},
 			{10, 6},
@@ -2335,6 +2335,19 @@ func TestStepIgnorePrivateRuntime(t *testing.T) {
 	// Tests that Step will ignore calls to private runtime functions
 	// (such as runtime.convT2E in this case)
 	switch {
+	case goversion.VersionAfterOrEqual(runtime.Version(), 1, 17) && (runtime.GOARCH == "amd64"):
+		testseq("teststepprog", contStep, []nextTest{
+			{21, 13},
+			{13, 14},
+			{14, 15},
+			{15, 17},
+			{17, 22}}, "", t)
+	case goversion.VersionAfterOrEqual(runtime.Version(), 1, 17):
+		testseq("teststepprog", contStep, []nextTest{
+			{21, 14},
+			{14, 15},
+			{15, 17},
+			{17, 22}}, "", t)
 	case goversion.VersionAfterOrEqual(runtime.Version(), 1, 11):
 		testseq("teststepprog", contStep, []nextTest{
 			{21, 14},
@@ -2598,7 +2611,7 @@ func TestStepOnCallPtrInstr(t *testing.T) {
 
 		assertNoError(p.Step(), t, "Step()")
 
-		if goversion.VersionAfterOrEqual(runtime.Version(), 1, 11) {
+		if goversion.VersionAfterOrEqual(runtime.Version(), 1, 11) && !(goversion.VersionAfterOrEqual(runtime.Version(), 1, 17) && (runtime.GOARCH == "amd64")) {
 			assertLineNumber(p, t, 6, "Step continued to wrong line,")
 		} else {
 			assertLineNumber(p, t, 5, "Step continued to wrong line,")
@@ -3153,7 +3166,7 @@ func TestIssue844(t *testing.T) {
 	})
 }
 
-func logStacktrace(t *testing.T, bi *proc.BinaryInfo, frames []proc.Stackframe) {
+func logStacktrace(t *testing.T, p *proc.Target, frames []proc.Stackframe) {
 	for j := range frames {
 		name := "?"
 		if frames[j].Current.Fn != nil {
@@ -3165,20 +3178,20 @@ func logStacktrace(t *testing.T, bi *proc.BinaryInfo, frames []proc.Stackframe)
 
 		t.Logf("\t%#x %#x %#x %s at %s:%d\n", frames[j].Call.PC, frames[j].FrameOffset(), frames[j].FramePointerOffset(), name, filepath.Base(frames[j].Call.File), frames[j].Call.Line)
 		if frames[j].TopmostDefer != nil {
-			f, l, fn := bi.PCToLine(frames[j].TopmostDefer.DeferredPC)
+			_, _, fn := frames[j].TopmostDefer.DeferredFunc(p)
 			fnname := ""
 			if fn != nil {
 				fnname = fn.Name
 			}
-			t.Logf("\t\ttopmost defer: %#x %s at %s:%d\n", frames[j].TopmostDefer.DeferredPC, fnname, f, l)
+			t.Logf("\t\ttopmost defer: %#x %s\n", frames[j].TopmostDefer.DwrapPC, fnname)
 		}
 		for deferIdx, _defer := range frames[j].Defers {
-			f, l, fn := bi.PCToLine(_defer.DeferredPC)
+			_, _, fn := _defer.DeferredFunc(p)
 			fnname := ""
 			if fn != nil {
 				fnname = fn.Name
 			}
-			t.Logf("\t\t%d defer: %#x %s at %s:%d\n", deferIdx, _defer.DeferredPC, fnname, f, l)
+			t.Logf("\t\t%d defer: %#x %s\n", deferIdx, _defer.DwrapPC, fnname)
 
 		}
 	}
@@ -3299,7 +3312,7 @@ func TestCgoStacktrace(t *testing.T) {
 			assertNoError(err, t, fmt.Sprintf("Stacktrace at iteration step %d", itidx))
 
 			t.Logf("iteration step %d", itidx)
-			logStacktrace(t, p.BinInfo(), frames)
+			logStacktrace(t, p, frames)
 
 			m := stacktraceCheck(t, tc, frames)
 			mismatch := (m == nil)
@@ -3336,7 +3349,7 @@ func TestCgoStacktrace(t *testing.T) {
 					if frames[j].Current.File != threadFrames[j].Current.File || frames[j].Current.Line != threadFrames[j].Current.Line {
 						t.Logf("stack mismatch between goroutine stacktrace and thread stacktrace")
 						t.Logf("thread stacktrace:")
-						logStacktrace(t, p.BinInfo(), threadFrames)
+						logStacktrace(t, p, threadFrames)
 						mismatch = true
 						break
 					}
@@ -3390,7 +3403,7 @@ func TestSystemstackStacktrace(t *testing.T) {
 		assertNoError(err, t, "GetG")
 		frames, err := g.Stacktrace(100, 0)
 		assertNoError(err, t, "stacktrace")
-		logStacktrace(t, p.BinInfo(), frames)
+		logStacktrace(t, p, frames)
 		m := stacktraceCheck(t, []string{"!runtime.startpanic_m", "runtime.gopanic", "main.main"}, frames)
 		if m == nil {
 			t.Fatal("see previous loglines")
@@ -3423,7 +3436,7 @@ func TestSystemstackOnRuntimeNewstack(t *testing.T) {
 		}
 		frames, err := g.Stacktrace(100, 0)
 		assertNoError(err, t, "stacktrace")
-		logStacktrace(t, p.BinInfo(), frames)
+		logStacktrace(t, p, frames)
 		m := stacktraceCheck(t, []string{"!runtime.newstack", "main.main"}, frames)
 		if m == nil {
 			t.Fatal("see previous loglines")
@@ -4047,7 +4060,7 @@ func TestReadDefer(t *testing.T) {
 		frames, err := p.SelectedGoroutine().Stacktrace(10, proc.StacktraceReadDefers)
 		assertNoError(err, t, "Stacktrace")
 
-		logStacktrace(t, p.BinInfo(), frames)
+		logStacktrace(t, p, frames)
 
 		examples := []struct {
 			frameIdx     int
@@ -4073,9 +4086,9 @@ func TestReadDefer(t *testing.T) {
 			if d.Unreadable != nil {
 				t.Fatalf("expected %q as %s of frame %d, got unreadable defer: %v", tgt, deferName, frameIdx, d.Unreadable)
 			}
-			dfn := p.BinInfo().PCToFunc(d.DeferredPC)
+			_, _, dfn := d.DeferredFunc(p)
 			if dfn == nil {
-				t.Fatalf("expected %q as %s of frame %d, got %#x", tgt, deferName, frameIdx, d.DeferredPC)
+				t.Fatalf("expected %q as %s of frame %d, got %#x", tgt, deferName, frameIdx, d.DwrapPC)
 			}
 			if dfn.Name != tgt {
 				t.Fatalf("expected %q as %s of frame %d, got %q", tgt, deferName, frameIdx, dfn.Name)
@@ -4113,6 +4126,19 @@ func TestNextUnknownInstr(t *testing.T) {
 }
 
 func TestReadDeferArgs(t *testing.T) {
+	if goversion.VersionAfterOrEqual(runtime.Version(), 1, 17) {
+		// When regabi is enabled in Go 1.17 and later, reading arguments of
+		// deferred functions becomes significantly more complicated because of
+		// the autogenerated code used to unpack the argument frame stored in
+		// runtime._defer into registers.
+		// We either need to know how to do the translation, implementing the ABI1
+		// rules in Delve, or have some assistence from the compiler (for example
+		// have the dwrap function contain entries for each of the captured
+		// variables with a location describing their offset from DX).
+		// Ultimately this feature is unimportant enough that we can leave it
+		// disabled for now.
+		t.Skip("unsupported")
+	}
 	var tests = []struct {
 		frame, deferCall int
 		a, b             int64
@@ -4128,9 +4154,13 @@ func TestReadDeferArgs(t *testing.T) {
 			scope, err := proc.ConvertEvalScope(p, -1, test.frame, test.deferCall)
 			assertNoError(err, t, fmt.Sprintf("ConvertEvalScope(-1, %d, %d)", test.frame, test.deferCall))
 
+			if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 17) {
+				// In Go 1.17 deferred function calls can end up inside a wrapper, and
+				// the scope for this evaluation needs to be the wrapper.
 				if scope.Fn.Name != "main.f2" {
 					t.Fatalf("expected function \"main.f2\" got %q", scope.Fn.Name)
 				}
+			}
 
 			avar, err := scope.EvalVariable("a", normalLoadConfig)
 			if err != nil {
@@ -4344,7 +4374,7 @@ func TestAncestors(t *testing.T) {
 			astack, err := a.Stack(100)
 			assertNoError(err, t, fmt.Sprintf("Ancestor %d stack", i))
 			t.Logf("ancestor %d\n", i)
-			logStacktrace(t, p.BinInfo(), astack)
+			logStacktrace(t, p, astack)
 			for _, frame := range astack {
 				if frame.Current.Fn != nil && frame.Current.Fn.Name == "main.main" {
 					mainFound = true
@@ -4480,7 +4510,7 @@ func TestCgoStacktrace2(t *testing.T) {
 		p.Continue()
 		frames, err := proc.ThreadStacktrace(p.CurrentThread(), 100)
 		assertNoError(err, t, "Stacktrace()")
-		logStacktrace(t, p.BinInfo(), frames)
+		logStacktrace(t, p, frames)
 		m := stacktraceCheck(t, []string{"C.sigsegv", "C.testfn", "main.main"}, frames)
 		if m == nil {
 			t.Fatal("see previous loglines")
@@ -4591,7 +4621,7 @@ func TestIssue1795(t *testing.T) {
 		assertNoError(p.Continue(), t, "Continue()")
 		frames, err := proc.ThreadStacktrace(p.CurrentThread(), 40)
 		assertNoError(err, t, "ThreadStacktrace()")
-		logStacktrace(t, p.BinInfo(), frames)
+		logStacktrace(t, p, frames)
 		if err := checkFrame(frames[0], "regexp.(*Regexp).doExecute", "", 0, false); err != nil {
 			t.Errorf("Wrong frame 0: %v", err)
 		}
@@ -5361,3 +5391,28 @@ func TestManualStopWhileStopped(t *testing.T) {
 		}
 	})
 }
+
+func TestDwrapStartLocation(t *testing.T) {
+	// Tests that the start location of a goroutine is unwrapped in Go 1.17 and later.
+	withTestProcess("goroutinestackprog", t, func(p *proc.Target, fixture protest.Fixture) {
+		setFunctionBreakpoint(p, t, "main.stacktraceme")
+		assertNoError(p.Continue(), t, "Continue()")
+		gs, _, err := proc.GoroutinesInfo(p, 0, 0)
+		assertNoError(err, t, "GoroutinesInfo")
+		found := false
+		for _, g := range gs {
+			startLoc := g.StartLoc(p)
+			if startLoc.Fn == nil {
+				continue
+			}
+			t.Logf("%#v\n", startLoc.Fn.Name)
+			if startLoc.Fn.Name == "main.agoroutine" {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Errorf("could not find any goroutine with a start location of main.agoroutine")
+		}
+	})
+}

pkg/proc/stack.go

@@ -519,7 +519,7 @@ func (it *stackIterator) loadG0SchedSP() {
 
 // Defer represents one deferred call
 type Defer struct {
-	DeferredPC uint64 // Value of field _defer.fn.fn, the deferred function
+	DwrapPC uint64 // Value of field _defer.fn.fn, the deferred function or a wrapper to it in Go 1.17 or later
 	DeferPC uint64 // PC address of instruction that added this defer
 	SP      uint64 // Value of SP register when this function was deferred (this field gets adjusted when the stack is moved to match the new stack space)
 	link    *Defer // Next deferred function
@@ -584,7 +584,7 @@ func (d *Defer) load() {
 	if fnvar.Addr != 0 {
 		fnvar = fnvar.loadFieldNamed("fn")
 		if fnvar.Unreadable == nil {
-			d.DeferredPC, _ = constant.Uint64Val(fnvar.Value)
+			d.DwrapPC, _ = constant.Uint64Val(fnvar.Value)
 		}
 	}
 
@@ -627,11 +627,11 @@ func (d *Defer) EvalScope(t *Target, thread Thread) (*EvalScope, error) {
 	}
 
 	bi := thread.BinInfo()
-	scope.PC = d.DeferredPC
+	scope.PC = d.DwrapPC
-	scope.File, scope.Line, scope.Fn = bi.PCToLine(d.DeferredPC)
+	scope.File, scope.Line, scope.Fn = bi.PCToLine(d.DwrapPC)
 
 	if scope.Fn == nil {
-		return nil, fmt.Errorf("could not find function at %#x", d.DeferredPC)
+		return nil, fmt.Errorf("could not find function at %#x", d.DwrapPC)
 	}
 
 	// The arguments are stored immediately after the defer header struct, i.e.
@@ -664,3 +664,16 @@ func (d *Defer) EvalScope(t *Target, thread Thread) (*EvalScope, error) {
 
 	return scope, nil
 }
+
+// DeferredFunc returns the deferred function, on Go 1.17 and later unwraps
+// any defer wrapper.
+func (d *Defer) DeferredFunc(p *Target) (file string, line int, fn *Function) {
+	bi := p.BinInfo()
+	fn = bi.PCToFunc(d.DwrapPC)
+	fn = p.dwrapUnwrap(fn)
+	if fn == nil {
+		return "", 0, nil
+	}
+	file, line = fn.cu.lineInfo.PCToLine(fn.Entry, fn.Entry)
+	return file, line, fn
+}

pkg/proc/target.go

@@ -465,3 +465,27 @@ func (t *Target) clearFakeMemory() {
 	t.fakeMemoryRegistry = t.fakeMemoryRegistry[:0]
 	t.fakeMemoryRegistryMap = make(map[string]*compositeMemory)
 }
+
+// dwrapUnwrap checks if fn is a dwrap wrapper function and unwraps it if it is.
+func (t *Target) dwrapUnwrap(fn *Function) *Function {
+	if fn == nil {
+		return nil
+	}
+	if !strings.Contains(fn.Name, "·dwrap·") {
+		return fn
+	}
+	if unwrap := t.BinInfo().dwrapUnwrapCache[fn.Entry]; unwrap != nil {
+		return unwrap
+	}
+	text, err := disassemble(t.Memory(), nil, t.Breakpoints(), t.BinInfo(), fn.Entry, fn.End, false)
+	if err != nil {
+		return fn
+	}
+	for _, instr := range text {
+		if instr.IsCall() && instr.DestLoc != nil && instr.DestLoc.Fn != nil && !instr.DestLoc.Fn.privateRuntime() {
+			t.BinInfo().dwrapUnwrapCache[fn.Entry] = instr.DestLoc.Fn
+			return instr.DestLoc.Fn
+		}
+	}
+	return fn
+}

pkg/proc/target_exec.go

@@ -2,6 +2,7 @@ package proc
 
 import (
 	"bytes"
+	"debug/dwarf"
 	"errors"
 	"fmt"
 	"go/ast"
@@ -275,7 +276,22 @@ func stepInstructionOut(dbp *Target, curthread Thread, fnname1, fnname2 string)
 			return err
 		}
 		loc, err := curthread.Location()
-		if err != nil || loc.Fn == nil || (loc.Fn.Name != fnname1 && loc.Fn.Name != fnname2) {
+		var locFnName string
+		if loc.Fn != nil {
+			locFnName = loc.Fn.Name
+			// Calls to runtime.Breakpoint are inlined in some versions of Go when
+			// inlining is enabled. Here we attempt to resolve any inlining.
+			dwarfTree, _ := loc.Fn.cu.image.getDwarfTree(loc.Fn.offset)
+			if dwarfTree != nil {
+				inlstack := reader.InlineStack(dwarfTree, loc.PC)
+				if len(inlstack) > 0 {
+					if locFnName2, ok := inlstack[0].Val(dwarf.AttrName).(string); ok {
+						locFnName = locFnName2
+					}
+				}
+			}
+		}
+		if err != nil || loc.Fn == nil || (locFnName != fnname1 && locFnName != fnname2) {
 			g, _ := GetG(curthread)
 			selg := dbp.SelectedGoroutine()
 			if g != nil && selg != nil && g.ID == selg.ID {
@@ -902,8 +918,8 @@ func skipAutogeneratedWrappersOut(g *G, thread Thread, startTopframe, startRetfr
 func setDeferBreakpoint(p *Target, text []AsmInstruction, topframe Stackframe, sameGCond ast.Expr, stepInto bool) (uint64, error) {
 	// Set breakpoint on the most recently deferred function (if any)
 	var deferpc uint64
-	if topframe.TopmostDefer != nil && topframe.TopmostDefer.DeferredPC != 0 {
+	if topframe.TopmostDefer != nil && topframe.TopmostDefer.DwrapPC != 0 {
-		deferfn := p.BinInfo().PCToFunc(topframe.TopmostDefer.DeferredPC)
+		_, _, deferfn := topframe.TopmostDefer.DeferredFunc(p)
 		var err error
 		deferpc, err = FirstPCAfterPrologue(p, deferfn, false)
 		if err != nil {
@@ -977,11 +993,15 @@ func stepOutReverse(p *Target, topframe, retframe Stackframe, sameGCond ast.Expr
 
 	var callpc uint64
 
-	if isPanicCall(frames) {
+	if ok, panicFrame := isPanicCall(frames); ok {
-		if len(frames) < 4 || frames[3].Current.Fn == nil {
+		if len(frames) < panicFrame+2 || frames[panicFrame+1].Current.Fn == nil {
-			return &ErrNoSourceForPC{frames[2].Current.PC}
+			if panicFrame < len(frames) {
+				return &ErrNoSourceForPC{frames[panicFrame].Current.PC}
+			} else {
+				return &ErrNoSourceForPC{frames[0].Current.PC}
 			}
-		callpc, err = findCallInstrForRet(p, p.Memory(), frames[2].Ret, frames[3].Current.Fn)
+		}
+		callpc, err = findCallInstrForRet(p, p.Memory(), frames[panicFrame].Ret, frames[panicFrame+1].Current.Fn)
 		if err != nil {
 			return err
 		}

pkg/proc/variables.go

@@ -516,15 +516,20 @@ func (g *G) Go() Location {
 }
 
 // StartLoc returns the starting location of the goroutine.
-func (g *G) StartLoc() Location {
+func (g *G) StartLoc(tgt *Target) Location {
-	f, l, fn := g.variable.bi.PCToLine(g.StartPC)
+	fn := g.variable.bi.PCToFunc(g.StartPC)
-	return Location{PC: g.StartPC, File: f, Line: l, Fn: fn}
+	fn = tgt.dwrapUnwrap(fn)
+	if fn == nil {
+		return Location{PC: g.StartPC}
+	}
+	f, l := fn.cu.lineInfo.PCToLine(fn.Entry, fn.Entry)
+	return Location{PC: fn.Entry, File: f, Line: l, Fn: fn}
 }
 
 // System returns true if g is a system goroutine. See isSystemGoroutine in
 // $GOROOT/src/runtime/traceback.go.
-func (g *G) System() bool {
+func (g *G) System(tgt *Target) bool {
-	loc := g.StartLoc()
+	loc := g.StartLoc(tgt)
 	if loc.Fn == nil {
 		return false
 	}

service/api/conversions.go

@@ -284,7 +284,7 @@ func ConvertFunction(fn *proc.Function) *Function {
 }
 
 // ConvertGoroutine converts from proc.G to api.Goroutine.
-func ConvertGoroutine(g *proc.G) *Goroutine {
+func ConvertGoroutine(tgt *proc.Target, g *proc.G) *Goroutine {
 	th := g.Thread
 	tid := 0
 	if th != nil {
@@ -298,7 +298,7 @@ func ConvertGoroutine(g *proc.G) *Goroutine {
 		CurrentLoc:     ConvertLocation(g.CurrentLoc),
 		UserCurrentLoc: ConvertLocation(g.UserCurrent()),
 		GoStatementLoc: ConvertLocation(g.Go()),
-		StartLoc:       ConvertLocation(g.StartLoc()),
+		StartLoc:       ConvertLocation(g.StartLoc(tgt)),
 		ThreadID:       tid,
 		WaitSince:      g.WaitSince,
 		WaitReason:     g.WaitReason,
@@ -308,10 +308,10 @@ func ConvertGoroutine(g *proc.G) *Goroutine {
 }
 
 // ConvertGoroutines converts from []*proc.G to []*api.Goroutine.
-func ConvertGoroutines(gs []*proc.G) []*Goroutine {
+func ConvertGoroutines(tgt *proc.Target, gs []*proc.G) []*Goroutine {
 	goroutines := make([]*Goroutine, len(gs))
 	for i := range gs {
-		goroutines[i] = ConvertGoroutine(gs[i])
+		goroutines[i] = ConvertGoroutine(tgt, gs[i])
 	}
 	return goroutines
 }

service/dap/server.go

@@ -1559,7 +1559,7 @@ func (s *Server) onStackTraceRequest(request *dap.StackTraceRequest) {
 	// Determine if the goroutine is a system goroutine.
 	isSystemGoroutine := true
 	if g, _ := s.debugger.FindGoroutine(goroutineID); g != nil {
-		isSystemGoroutine = g.System()
+		isSystemGoroutine = g.System(s.debugger.Target())
 	}
 
 	stackFrames := make([]dap.StackFrame, len(frames))

service/dap/server_test.go

@@ -2775,9 +2775,14 @@ func TestWorkingDir(t *testing.T) {
 					client.VariablesRequest(1001) // Locals
 					locals := client.ExpectVariablesResponse(t)
 					checkChildren(t, locals, "Locals", 2)
-					checkVarExact(t, locals, 0, "pwd", "pwd", fmt.Sprintf("%q", wd), "string", noChildren)
+					for i := range locals.Body.Variables {
-					checkVarExact(t, locals, 1, "err", "err", "error nil", "error", noChildren)
+						switch locals.Body.Variables[i].Name {
-
+						case "pwd":
+							checkVarExact(t, locals, i, "pwd", "pwd", fmt.Sprintf("%q", wd), "string", noChildren)
+						case "err":
+							checkVarExact(t, locals, i, "err", "err", "error nil", "error", noChildren)
+						}
+					}
 				},
 				disconnect: false,
 			}})
@@ -3210,7 +3215,7 @@ func TestEvaluateCallRequest(t *testing.T) {
 				disconnect: false,
 			}, { // Stop at runtime breakpoint
 				execute: func() {
-					checkStop(t, client, 1, "main.main", 197)
+					checkStop(t, client, 1, "main.main", -1)
 
 					// No return values
 					client.EvaluateRequest("call call0(1, 2)", 1000, "this context will be ignored")
@@ -3501,13 +3506,19 @@ func TestStepOutPreservesGoroutine(t *testing.T) {
 					if len(bestg) > 0 {
 						goroutineId = bestg[rand.Intn(len(bestg))]
 						t.Logf("selected goroutine %d (best)\n", goroutineId)
-					} else {
+					} else if len(candg) > 0 {
 						goroutineId = candg[rand.Intn(len(candg))]
 						t.Logf("selected goroutine %d\n", goroutineId)
 
 					}
+
+					if goroutineId != 0 {
 						client.StepOutRequest(goroutineId)
 						client.ExpectStepOutResponse(t)
+					} else {
+						client.ContinueRequest(-1)
+						client.ExpectContinueResponse(t)
+					}
 
 					switch e := client.ExpectMessage(t).(type) {
 					case *dap.StoppedEvent:
@@ -4490,7 +4501,7 @@ func TestSetVariableWithCall(t *testing.T) {
 				execute: func() {
 					tester := &helperForSetVariable{t, client}
 
-					checkStop(t, client, 1, "main.main", 197)
+					checkStop(t, client, 1, "main.main", -1)
 
 					_ = tester.variables(1001)
 
@@ -4499,7 +4510,7 @@ func TestSetVariableWithCall(t *testing.T) {
 					tester.evaluateRegex("str", `.*in main.callstacktrace at.*`, noChildren)
 
 					tester.failSetVariableAndStop(1001, "str", `callpanic()`, `callpanic panicked`)
-					checkStop(t, client, 1, "main.main", 197)
+					checkStop(t, client, 1, "main.main", -1)
 
 					// breakpoint during a function call.
 					tester.failSetVariableAndStop(1001, "str", `callbreak()`, "call stopped")

service/debugger/debugger.go

@@ -590,7 +590,7 @@ func (d *Debugger) state(retLoadCfg *proc.LoadConfig) (*api.DebuggerState, error
 	)
 
 	if d.target.SelectedGoroutine() != nil {
-		goroutine = api.ConvertGoroutine(d.target.SelectedGoroutine())
+		goroutine = api.ConvertGoroutine(d.target, d.target.SelectedGoroutine())
 	}
 
 	exited := false
@@ -1269,7 +1269,7 @@ func (d *Debugger) collectBreakpointInformation(state *api.DebuggerState) error
 			if err != nil {
 				return err
 			}
-			bpi.Goroutine = api.ConvertGoroutine(g)
+			bpi.Goroutine = api.ConvertGoroutine(d.target, g)
 		}
 
 		if bp.Stacktrace > 0 {
@@ -1536,7 +1536,7 @@ func (d *Debugger) FilterGoroutines(gs []*proc.G, filters []api.ListGoroutinesFi
 	for _, g := range gs {
 		ok := true
 		for i := range filters {
-			if !matchGoroutineFilter(g, &filters[i]) {
+			if !matchGoroutineFilter(d.target, g, &filters[i]) {
 				ok = false
 				break
 			}
@@ -1548,7 +1548,7 @@ func (d *Debugger) FilterGoroutines(gs []*proc.G, filters []api.ListGoroutinesFi
 	return r
 }
 
-func matchGoroutineFilter(g *proc.G, filter *api.ListGoroutinesFilter) bool {
+func matchGoroutineFilter(tgt *proc.Target, g *proc.G, filter *api.ListGoroutinesFilter) bool {
 	var val bool
 	switch filter.Kind {
 	default:
@@ -1562,7 +1562,7 @@ func matchGoroutineFilter(g *proc.G, filter *api.ListGoroutinesFilter) bool {
 	case api.GoroutineGoLoc:
 		val = matchGoroutineLocFilter(g.Go(), filter.Arg)
 	case api.GoroutineStartLoc:
-		val = matchGoroutineLocFilter(g.StartLoc(), filter.Arg)
+		val = matchGoroutineLocFilter(g.StartLoc(tgt), filter.Arg)
 	case api.GoroutineLabel:
 		idx := strings.Index(filter.Arg, "=")
 		if idx >= 0 {
@@ -1573,7 +1573,7 @@ func matchGoroutineFilter(g *proc.G, filter *api.ListGoroutinesFilter) bool {
 	case api.GoroutineRunning:
 		val = g.Thread != nil
 	case api.GoroutineUser:
-		val = !g.System()
+		val = !g.System(tgt)
 	}
 	if filter.Negated {
 		val = !val
@@ -1616,13 +1616,13 @@ func (d *Debugger) GroupGoroutines(gs []*proc.G, group *api.GoroutineGroupingOpt
 		case api.GoroutineGoLoc:
 			key = formatLoc(g.Go())
 		case api.GoroutineStartLoc:
-			key = formatLoc(g.StartLoc())
+			key = formatLoc(g.StartLoc(d.target))
 		case api.GoroutineLabel:
 			key = fmt.Sprintf("%s=%s", group.GroupByKey, g.Labels()[group.GroupByKey])
 		case api.GoroutineRunning:
 			key = fmt.Sprintf("running=%v", g.Thread != nil)
 		case api.GoroutineUser:
-			key = fmt.Sprintf("user=%v", !g.System())
+			key = fmt.Sprintf("user=%v", !g.System(d.target))
 		}
 		if len(groupMembers[key]) < group.MaxGroupMembers {
 			groupMembers[key] = append(groupMembers[key], g)
@@ -1764,12 +1764,12 @@ func (d *Debugger) convertStacktrace(rawlocs []proc.Stackframe, cfg *proc.LoadCo
 func (d *Debugger) convertDefers(defers []*proc.Defer) []api.Defer {
 	r := make([]api.Defer, len(defers))
 	for i := range defers {
-		ddf, ddl, ddfn := d.target.BinInfo().PCToLine(defers[i].DeferredPC)
+		ddf, ddl, ddfn := defers[i].DeferredFunc(d.target)
 		drf, drl, drfn := d.target.BinInfo().PCToLine(defers[i].DeferPC)
 
 		r[i] = api.Defer{
 			DeferredLoc: api.ConvertLocation(proc.Location{
-				PC:   defers[i].DeferredPC,
+				PC:   ddfn.Entry,
 				File: ddf,
 				Line: ddl,
 				Fn:   ddfn,
@@ -2109,6 +2109,10 @@ func (d *Debugger) DumpCancel() error {
 	return nil
 }
 
+func (d *Debugger) Target() *proc.Target {
+	return d.target
+}
+
 func go11DecodeErrorCheck(err error) error {
 	if _, isdecodeerr := err.(dwarf.DecodeError); !isdecodeerr {
 		return err

service/rpc1/server.go

@@ -288,7 +288,7 @@ func (s *RPCServer) ListGoroutines(arg interface{}, goroutines *[]*api.Goroutine
 	}
 	s.debugger.LockTarget()
 	s.debugger.UnlockTarget()
-	*goroutines = api.ConvertGoroutines(gs)
+	*goroutines = api.ConvertGoroutines(s.debugger.Target(), gs)
 	return nil
 }
 

service/rpc2/server.go

@@ -636,7 +636,7 @@ func (s *RPCServer) ListGoroutines(arg ListGoroutinesIn, out *ListGoroutinesOut)
 	gs, out.Groups, out.TooManyGroups = s.debugger.GroupGoroutines(gs, &arg.GoroutineGroupingOptions)
 	s.debugger.LockTarget()
 	defer s.debugger.UnlockTarget()
-	out.Goroutines = api.ConvertGoroutines(gs)
+	out.Goroutines = api.ConvertGoroutines(s.debugger.Target(), gs)
 	out.Nextg = nextg
 	return nil
 }

service/test/integration2_test.go

@@ -1891,17 +1891,9 @@ func TestAcceptMulticlient(t *testing.T) {
 	<-serverDone
 }
 
-func mustHaveDebugCalls(t *testing.T, c service.Client) {
-	locs, err := c.FindLocation(api.EvalScope{GoroutineID: -1}, "runtime.debugCallV1", false, nil)
-	if len(locs) == 0 || err != nil {
-		t.Skip("function calls not supported on this version of go")
-	}
-}
-
 func TestClientServerFunctionCall(t *testing.T) {
 	protest.MustSupportFunctionCalls(t, testBackend)
 	withTestClient2("fncall", t, func(c service.Client) {
-		mustHaveDebugCalls(t, c)
 		c.SetReturnValuesLoadConfig(&normalLoadConfig)
 		state := <-c.Continue()
 		assertNoError(state.Err, t, "Continue()")
@@ -1935,7 +1927,6 @@ func TestClientServerFunctionCallBadPos(t *testing.T) {
 		t.Skip("this is a safe point for Go 1.12")
 	}
 	withTestClient2("fncall", t, func(c service.Client) {
-		mustHaveDebugCalls(t, c)
 		loc, err := c.FindLocation(api.EvalScope{GoroutineID: -1}, "fmt/print.go:649", false, nil)
 		assertNoError(err, t, "could not find location")
 
@@ -1959,7 +1950,6 @@ func TestClientServerFunctionCallBadPos(t *testing.T) {
 func TestClientServerFunctionCallPanic(t *testing.T) {
 	protest.MustSupportFunctionCalls(t, testBackend)
 	withTestClient2("fncall", t, func(c service.Client) {
-		mustHaveDebugCalls(t, c)
 		c.SetReturnValuesLoadConfig(&normalLoadConfig)
 		state := <-c.Continue()
 		assertNoError(state.Err, t, "Continue()")
@@ -1988,7 +1978,6 @@ func TestClientServerFunctionCallStacktrace(t *testing.T) {
 	}
 	protest.MustSupportFunctionCalls(t, testBackend)
 	withTestClient2("fncall", t, func(c service.Client) {
-		mustHaveDebugCalls(t, c)
 		c.SetReturnValuesLoadConfig(&api.LoadConfig{FollowPointers: false, MaxStringLen: 2048})
 		state := <-c.Continue()
 		assertNoError(state.Err, t, "Continue()")

service/test/variables_test.go

@@ -1283,12 +1283,12 @@ func TestCallFunction(t *testing.T) {
 		{`strings.Join(s1, comma)`, nil, errors.New(`error evaluating "s1" as argument elems in function strings.Join: could not find symbol value for s1`)},
 	}
 
-	withTestProcess("fncall", t, func(p *proc.Target, fixture protest.Fixture) {
+	var testcases117 = []testCaseCallFunction{
-		_, err := proc.FindFunctionLocation(p, "runtime.debugCallV1", 0)
+		{`regabistacktest("one", "two", "three", "four", "five", 4)`, []string{`:string:"onetwo"`, `:string:"twothree"`, `:string:"threefour"`, `:string:"fourfive"`, `:string:"fiveone"`, ":uint8:8"}, nil},
-		if err != nil {
+		{`regabistacktest2(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)`, []string{":int:3", ":int:5", ":int:7", ":int:9", ":int:11", ":int:13", ":int:15", ":int:17", ":int:19", ":int:11"}, nil},
-			t.Skip("function calls not supported on this version of go")
 	}
 
+	withTestProcessArgs("fncall", t, ".", nil, protest.AllNonOptimized, func(p *proc.Target, fixture protest.Fixture) {
 		testCallFunctionSetBreakpoint(t, p, fixture)
 
 		assertNoError(p.Continue(), t, "Continue()")
@@ -1319,6 +1319,12 @@ func TestCallFunction(t *testing.T) {
 			}
 		}
 
+		if goversion.VersionAfterOrEqual(runtime.Version(), 1, 17) {
+			for _, tc := range testcases117 {
+				testCallFunction(t, p, tc)
+			}
+		}
+
 		// LEAVE THIS AS THE LAST ITEM, IT BREAKS THE TARGET PROCESS!!!
 		testCallFunction(t, p, testCaseCallFunction{"-unsafe escapeArg(&a2)", nil, nil})
 	})