proc: give unique addresses to registerized variables (#2527)

We told clients that further loading of variables can be done by
specifying a type cast using the address of a variable that we
returned.
This does not work for registerized variables (or, in general,
variables that have a complex location expression) because we don't
give them unique addresses and we throw away the compositeMemory object
we made to read them.

This commit changes proc so that:

1. variables with location expression divided in pieces do get a unique
   memory address
2. the compositeMemory object is saved somewhere
3. when an integer is cast back into a pointer type we look through our
   saved compositeMemory objects to see if there is one that covers the
   specified address and use it.

The unique memory addresses we generate have the MSB set to 1, as
specified by the Intel 86x64 manual addresses in this form are reserved
for kernel memory (which we can not read anyway) so we are guaranteed
to never generate a fake memory address that overlaps a real memory
address of the application.

The unfortunate side effect of this is that it will break clients that
do not deserialize the address to a 64bit integer. This practice is
contrary to how we defined our types and contrary to the specification
of the JSON format, as of json.org, however it is also fairly common,
due to javascript itself having only 53bit integers.

We could come up with a new mechanism but then even more old clients
would have to be changed.

_fixtures/morestringarg.go

@@ -0,0 +1,11 @@
+package main
+
+import "fmt"
+
+func f(s, q string) {
+	fmt.Println(s)
+}
+
+func main() {
+	f("very long string 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789X", "very long string B 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789X2")
+}

pkg/proc/breakpoints.go

@@ -272,9 +272,9 @@ func evalBreakpointCondition(thread Thread, cond ast.Expr) (bool, error) {
 	if cond == nil {
 		return true, nil
 	}
-	scope, err := GoroutineScope(thread)
+	scope, err := GoroutineScope(nil, thread)
 	if err != nil {
-		scope, err = ThreadScope(thread)
+		scope, err = ThreadScope(nil, thread)
 		if err != nil {
 			return true, err
 		}
@@ -568,7 +568,7 @@ func configureReturnBreakpoint(bi *BinaryInfo, bp *Breakpoint, topframe *Stackfr
 	}
 }
 
-func (rbpi *returnBreakpointInfo) Collect(thread Thread) []*Variable {
+func (rbpi *returnBreakpointInfo) Collect(t *Target, thread Thread) []*Variable {
 	if rbpi == nil {
 		return nil
 	}
@@ -577,7 +577,7 @@ func (rbpi *returnBreakpointInfo) Collect(thread Thread) []*Variable {
 	if err != nil {
 		return returnInfoError("could not get g", err, thread.ProcessMemory())
 	}
-	scope, err := GoroutineScope(thread)
+	scope, err := GoroutineScope(t, thread)
 	if err != nil {
 		return returnInfoError("could not get scope", err, thread.ProcessMemory())
 	}

pkg/proc/core/core_test.go

@@ -297,7 +297,7 @@ func TestCore(t *testing.T) {
 	if mainFrame == nil {
 		t.Fatalf("Couldn't find main in stack %v", panickingStack)
 	}
-	msg, err := proc.FrameToScope(p.BinInfo(), p.Memory(), nil, *mainFrame).EvalVariable("msg", proc.LoadConfig{MaxStringLen: 64})
+	msg, err := proc.FrameToScope(p, p.BinInfo(), p.Memory(), nil, *mainFrame).EvalVariable("msg", proc.LoadConfig{MaxStringLen: 64})
 	if err != nil {
 		t.Fatalf("Couldn't EvalVariable(msg, ...): %v", err)
 	}
@@ -427,7 +427,7 @@ mainSearch:
 		t.Fatal("could not find main.main frame")
 	}
 
-	scope := proc.FrameToScope(p.BinInfo(), p.Memory(), nil, *mainFrame)
+	scope := proc.FrameToScope(p, p.BinInfo(), p.Memory(), nil, *mainFrame)
 	loadConfig := proc.LoadConfig{FollowPointers: true, MaxVariableRecurse: 1, MaxStringLen: 64, MaxArrayValues: 64, MaxStructFields: -1}
 	v1, err := scope.EvalVariable("t", loadConfig)
 	assertNoError(err, t, "EvalVariable(t)")

pkg/proc/dwarf_export_test.go

@@ -7,7 +7,7 @@ func (bi *BinaryInfo) PackageVars() []packageVar {
 	return bi.packageVars
 }
 
-func NewCompositeMemory(p *Target, pieces []op.Piece) (*compositeMemory, error) {
+func NewCompositeMemory(p *Target, pieces []op.Piece, base uint64) (*compositeMemory, error) {
 	regs, err := p.CurrentThread().Registers()
 	if err != nil {
 		return nil, err
@@ -17,5 +17,9 @@ func NewCompositeMemory(p *Target, pieces []op.Piece) (*compositeMemory, error)
 	dwarfregs := arch.RegistersToDwarfRegisters(0, regs)
 	dwarfregs.ChangeFunc = p.CurrentThread().SetReg
 
-	return newCompositeMemory(p.Memory(), arch, *dwarfregs, pieces)
+	mem, err := newCompositeMemory(p.Memory(), arch, *dwarfregs, pieces)
+	if mem != nil {
+		mem.base = base
+	}
+	return mem, err
 }

pkg/proc/eval.go

@@ -32,6 +32,7 @@ type EvalScope struct {
 	Mem     MemoryReadWriter // Target's memory
 	g       *G
 	BinInfo *BinaryInfo
+	target  *Target
 
 	frameOffset int64
 
@@ -68,7 +69,7 @@ func ConvertEvalScope(dbp *Target, gid, frame, deferCall int) (*EvalScope, error
 		return nil, err
 	}
 	if g == nil {
-		return ThreadScope(ct)
+		return ThreadScope(dbp, ct)
 	}
 
 	var opts StacktraceOptions
@@ -95,10 +96,10 @@ func ConvertEvalScope(dbp *Target, gid, frame, deferCall int) (*EvalScope, error
 			return nil, d.Unreadable
 		}
 
-		return d.EvalScope(ct)
+		return d.EvalScope(dbp, ct)
 	}
 
-	return FrameToScope(dbp.BinInfo(), dbp.Memory(), g, locs[frame:]...), nil
+	return FrameToScope(dbp, dbp.BinInfo(), dbp.Memory(), g, locs[frame:]...), nil
 }
 
 // FrameToScope returns a new EvalScope for frames[0].
@@ -106,7 +107,7 @@ func ConvertEvalScope(dbp *Target, gid, frame, deferCall int) (*EvalScope, error
 // frames[0].Regs.SP() and frames[1].Regs.CFA will be cached.
 // Otherwise all memory between frames[0].Regs.SP() and frames[0].Regs.CFA
 // will be cached.
-func FrameToScope(bi *BinaryInfo, thread MemoryReadWriter, g *G, frames ...Stackframe) *EvalScope {
+func FrameToScope(t *Target, bi *BinaryInfo, thread MemoryReadWriter, g *G, frames ...Stackframe) *EvalScope {
 	// Creates a cacheMem that will preload the entire stack frame the first
 	// time any local variable is read.
 	// Remember that the stack grows downward in memory.
@@ -121,13 +122,13 @@ func FrameToScope(bi *BinaryInfo, thread MemoryReadWriter, g *G, frames ...Stack
 		thread = cacheMemory(thread, minaddr, int(maxaddr-minaddr))
 	}
 
-	s := &EvalScope{Location: frames[0].Call, Regs: frames[0].Regs, Mem: thread, g: g, BinInfo: bi, frameOffset: frames[0].FrameOffset()}
+	s := &EvalScope{Location: frames[0].Call, Regs: frames[0].Regs, Mem: thread, g: g, BinInfo: bi, target: t, frameOffset: frames[0].FrameOffset()}
 	s.PC = frames[0].lastpc
 	return s
 }
 
 // ThreadScope returns an EvalScope for the given thread.
-func ThreadScope(thread Thread) (*EvalScope, error) {
+func ThreadScope(t *Target, thread Thread) (*EvalScope, error) {
 	locations, err := ThreadStacktrace(thread, 1)
 	if err != nil {
 		return nil, err
@@ -135,11 +136,11 @@ func ThreadScope(thread Thread) (*EvalScope, error) {
 	if len(locations) < 1 {
 		return nil, errors.New("could not decode first frame")
 	}
-	return FrameToScope(thread.BinInfo(), thread.ProcessMemory(), nil, locations...), nil
+	return FrameToScope(t, thread.BinInfo(), thread.ProcessMemory(), nil, locations...), nil
 }
 
 // GoroutineScope returns an EvalScope for the goroutine running on the given thread.
-func GoroutineScope(thread Thread) (*EvalScope, error) {
+func GoroutineScope(t *Target, thread Thread) (*EvalScope, error) {
 	locations, err := ThreadStacktrace(thread, 1)
 	if err != nil {
 		return nil, err
@@ -151,7 +152,7 @@ func GoroutineScope(thread Thread) (*EvalScope, error) {
 	if err != nil {
 		return nil, err
 	}
-	return FrameToScope(thread.BinInfo(), thread.ProcessMemory(), g, locations...), nil
+	return FrameToScope(t, thread.BinInfo(), thread.ProcessMemory(), g, locations...), nil
 }
 
 // EvalExpression returns the value of the given expression.
@@ -219,7 +220,7 @@ func (scope *EvalScope) Locals() ([]*Variable, error) {
 	vars := make([]*Variable, 0, len(varEntries))
 	depths := make([]int, 0, len(varEntries))
 	for _, entry := range varEntries {
-		val, err := extractVarInfoFromEntry(scope.BinInfo, scope.image(), scope.Regs, scope.Mem, entry.Tree)
+		val, err := extractVarInfoFromEntry(scope.target, scope.BinInfo, scope.image(), scope.Regs, scope.Mem, entry.Tree)
 		if err != nil {
 			// skip variables that we can't parse yet
 			continue
@@ -472,7 +473,7 @@ func (scope *EvalScope) PackageVariables(cfg LoadConfig) ([]*Variable, error) {
 		}
 
 		// Ignore errors trying to extract values
-		val, err := extractVarInfoFromEntry(scope.BinInfo, pkgvar.cu.image, regsReplaceStaticBase(scope.Regs, pkgvar.cu.image), scope.Mem, godwarf.EntryToTree(entry))
+		val, err := extractVarInfoFromEntry(scope.target, scope.BinInfo, pkgvar.cu.image, regsReplaceStaticBase(scope.Regs, pkgvar.cu.image), scope.Mem, godwarf.EntryToTree(entry))
 		if val != nil && val.Kind == reflect.Invalid {
 			continue
 		}
@@ -509,7 +510,7 @@ func (scope *EvalScope) findGlobalInternal(name string) (*Variable, error) {
 			if err != nil {
 				return nil, err
 			}
-			return extractVarInfoFromEntry(scope.BinInfo, pkgvar.cu.image, regsReplaceStaticBase(scope.Regs, pkgvar.cu.image), scope.Mem, godwarf.EntryToTree(entry))
+			return extractVarInfoFromEntry(scope.target, scope.BinInfo, pkgvar.cu.image, regsReplaceStaticBase(scope.Regs, pkgvar.cu.image), scope.Mem, godwarf.EntryToTree(entry))
 		}
 	}
 	for _, fn := range scope.BinInfo.Functions {
@@ -720,7 +721,7 @@ func (scope *EvalScope) evalAST(t ast.Expr) (*Variable, error) {
 					if err != nil {
 						return nil, fmt.Errorf("blah: %v", err)
 					}
-					gvar := newVariable("curg", fakeAddress, typ, scope.BinInfo, scope.Mem)
+					gvar := newVariable("curg", fakeAddressUnresolv, typ, scope.BinInfo, scope.Mem)
 					gvar.loaded = true
 					gvar.Flags = VariableFakeAddress
 					gvar.Children = append(gvar.Children, *newConstant(constant.MakeInt64(0), scope.Mem))
@@ -841,7 +842,14 @@ func (scope *EvalScope) evalTypeCast(node *ast.CallExpr) (*Variable, error) {
 
 		n, _ := constant.Int64Val(argv.Value)
 
-		v.Children = []Variable{*(newVariable("", uint64(n), ttyp.Type, scope.BinInfo, scope.Mem))}
+		mem := scope.Mem
+		if scope.target != nil {
+			if mem2 := scope.target.findFakeMemory(uint64(n)); mem2 != nil {
+				mem = mem2
+			}
+		}
+
+		v.Children = []Variable{*(newVariable("", uint64(n), ttyp.Type, scope.BinInfo, mem))}
 		v.Children[0].OnlyAddr = true
 		return v, nil
 
@@ -1158,9 +1166,9 @@ func (scope *EvalScope) evalIdent(node *ast.Ident) (*Variable, error) {
 				v := newVariable(node.Name, 0, typ, scope.BinInfo, scope.Mem)
 				if v.Kind == reflect.String {
 					v.Len = int64(len(reg.Bytes) * 2)
-					v.Base = fakeAddress
+					v.Base = fakeAddressUnresolv
 				}
-				v.Addr = fakeAddress
+				v.Addr = fakeAddressUnresolv
 				v.Flags = VariableCPURegister
 				v.reg = reg
 				return v, nil

pkg/proc/fncall.go

@@ -168,7 +168,7 @@ func EvalExpressionWithCalls(t *Target, g *G, expr string, retLoadCfg LoadConfig
 		return errFuncCallUnsupported
 	}
 
-	scope, err := GoroutineScope(g.Thread)
+	scope, err := GoroutineScope(t, g.Thread)
 	if err != nil {
 		return err
 	}
@@ -722,7 +722,7 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 	switch rax {
 	case debugCallAXPrecheckFailed:
 		// get error from top of the stack and return it to user
-		errvar, err := readTopstackVariable(thread, regs, "string", loadFullValue)
+		errvar, err := readTopstackVariable(p, thread, regs, "string", loadFullValue)
 		if err != nil {
 			fncall.err = fmt.Errorf("could not get precheck error reason: %v", err)
 			break
@@ -797,7 +797,7 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 		if fncall.panicvar != nil || fncall.lateCallFailure {
 			break
 		}
-		retScope, err := ThreadScope(thread)
+		retScope, err := ThreadScope(p, thread)
 		if err != nil {
 			fncall.err = fmt.Errorf("could not get return values: %v", err)
 			break
@@ -830,7 +830,7 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 
 	case debugCallAXReadPanic:
 		// read panic value from stack
-		fncall.panicvar, err = readTopstackVariable(thread, regs, "interface {}", callScope.callCtx.retLoadCfg)
+		fncall.panicvar, err = readTopstackVariable(p, thread, regs, "interface {}", callScope.callCtx.retLoadCfg)
 		if err != nil {
 			fncall.err = fmt.Errorf("could not get panic: %v", err)
 			break
@@ -846,9 +846,9 @@ func funcCallStep(callScope *EvalScope, fncall *functionCallState, thread Thread
 	return false
 }
 
-func readTopstackVariable(thread Thread, regs Registers, typename string, loadCfg LoadConfig) (*Variable, error) {
+func readTopstackVariable(t *Target, thread Thread, regs Registers, typename string, loadCfg LoadConfig) (*Variable, error) {
 	bi := thread.BinInfo()
-	scope, err := ThreadScope(thread)
+	scope, err := ThreadScope(t, thread)
 	if err != nil {
 		return nil, err
 	}

pkg/proc/mem.go

@@ -76,11 +76,6 @@ func cacheMemory(mem MemoryReadWriter, addr uint64, size int) MemoryReadWriter {
 	return &memCache{false, addr, make([]byte, size), mem}
 }
 
-// fakeAddress used by extractVarInfoFromEntry for variables that do not
-// have a memory address, we can't use 0 because a lot of code (likely
-// including client code) assumes that addr == 0 is nil
-const fakeAddress = 0xbeef0000
-
 // compositeMemory represents a chunk of memory that is stored in CPU
 // registers or non-contiguously.
 //
@@ -89,6 +84,7 @@ const fakeAddress = 0xbeef0000
 // with some fields stored into CPU registers and other fields stored in
 // memory.
 type compositeMemory struct {
+	base    uint64 // base address for this composite memory
 	realmem MemoryReadWriter
 	arch    *Arch
 	regs    op.DwarfRegisters
@@ -133,7 +129,7 @@ func newCompositeMemory(mem MemoryReadWriter, arch *Arch, regs op.DwarfRegisters
 }
 
 func (mem *compositeMemory) ReadMemory(data []byte, addr uint64) (int, error) {
-	addr -= fakeAddress
+	addr -= mem.base
 	if addr >= uint64(len(mem.data)) || addr+uint64(len(data)) > uint64(len(mem.data)) {
 		return 0, errors.New("read out of bounds")
 	}
@@ -142,7 +138,7 @@ func (mem *compositeMemory) ReadMemory(data []byte, addr uint64) (int, error) {
 }
 
 func (mem *compositeMemory) WriteMemory(addr uint64, data []byte) (int, error) {
-	addr -= fakeAddress
+	addr -= mem.base
 	if addr >= uint64(len(mem.data)) || addr+uint64(len(data)) > uint64(len(mem.data)) {
 		return 0, errors.New("write out of bounds")
 	}

pkg/proc/proc_test.go

@@ -1149,10 +1149,10 @@ func evalVariableOrError(p *proc.Target, symbol string) (*proc.Variable, error)
 		var frame proc.Stackframe
 		frame, err = findFirstNonRuntimeFrame(p)
 		if err == nil {
-			scope = proc.FrameToScope(p.BinInfo(), p.Memory(), nil, frame)
+			scope = proc.FrameToScope(p, p.BinInfo(), p.Memory(), nil, frame)
 		}
 	} else {
-		scope, err = proc.GoroutineScope(p.CurrentThread())
+		scope, err = proc.GoroutineScope(p, p.CurrentThread())
 	}
 
 	if err != nil {
@@ -1172,7 +1172,7 @@ func evalVariable(p *proc.Target, t testing.TB, symbol string) *proc.Variable {
 }
 
 func setVariable(p *proc.Target, symbol, value string) error {
-	scope, err := proc.GoroutineScope(p.CurrentThread())
+	scope, err := proc.GoroutineScope(p, p.CurrentThread())
 	if err != nil {
 		return err
 	}
@@ -1350,7 +1350,7 @@ func TestPointerSetting(t *testing.T) {
 		pval(1)
 
 		// change p1 to point to i2
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, t, "Scope()")
 		i2addr, err := scope.EvalExpression("i2", normalLoadConfig)
 		assertNoError(err, t, "EvalExpression()")
@@ -1481,7 +1481,7 @@ func TestBreakpointCountsWithDetection(t *testing.T) {
 				if bp := th.Breakpoint(); bp.Breakpoint == nil {
 					continue
 				}
-				scope, err := proc.GoroutineScope(th)
+				scope, err := proc.GoroutineScope(p, th)
 				assertNoError(err, t, "Scope()")
 				v, err := scope.EvalVariable("i", normalLoadConfig)
 				assertNoError(err, t, "evalVariable")
@@ -1551,7 +1551,7 @@ func BenchmarkGoroutinesInfo(b *testing.B) {
 		assertNoError(p.Continue(), b, "Continue()")
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			p.ClearAllGCache()
+			p.ClearCaches()
 			_, _, err := proc.GoroutinesInfo(p, 0, 0)
 			assertNoError(err, b, "GoroutinesInfo")
 		}
@@ -1611,7 +1611,7 @@ func TestPointerLoops(t *testing.T) {
 func BenchmarkLocalVariables(b *testing.B) {
 	withTestProcess("testvariables", b, func(p *proc.Target, fixture protest.Fixture) {
 		assertNoError(p.Continue(), b, "Continue() returned an error")
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, b, "Scope()")
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
@@ -1900,7 +1900,7 @@ func TestPackageVariables(t *testing.T) {
 	withTestProcess("testvariables", t, func(p *proc.Target, fixture protest.Fixture) {
 		err := p.Continue()
 		assertNoError(err, t, "Continue()")
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, t, "Scope()")
 		vars, err := scope.PackageVariables(normalLoadConfig)
 		assertNoError(err, t, "PackageVariables()")
@@ -2698,7 +2698,7 @@ func BenchmarkTrace(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			assertNoError(p.Continue(), b, "Continue()")
-			s, err := proc.GoroutineScope(p.CurrentThread())
+			s, err := proc.GoroutineScope(p, p.CurrentThread())
 			assertNoError(err, b, "Scope()")
 			_, err = s.FunctionArguments(proc.LoadConfig{false, 0, 64, 0, 3, 0})
 			assertNoError(err, b, "FunctionArguments()")
@@ -2966,10 +2966,10 @@ func TestIssue871(t *testing.T) {
 			var frame proc.Stackframe
 			frame, err = findFirstNonRuntimeFrame(p)
 			if err == nil {
-				scope = proc.FrameToScope(p.BinInfo(), p.Memory(), nil, frame)
+				scope = proc.FrameToScope(p, p.BinInfo(), p.Memory(), nil, frame)
 			}
 		} else {
-			scope, err = proc.GoroutineScope(p.CurrentThread())
+			scope, err = proc.GoroutineScope(p, p.CurrentThread())
 		}
 		assertNoError(err, t, "scope")
 
@@ -3007,7 +3007,7 @@ func TestShadowedFlag(t *testing.T) {
 	}
 	withTestProcess("testshadow", t, func(p *proc.Target, fixture protest.Fixture) {
 		assertNoError(p.Continue(), t, "Continue")
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, t, "GoroutineScope")
 		locals, err := scope.LocalVariables(normalLoadConfig)
 		assertNoError(err, t, "LocalVariables")
@@ -3412,7 +3412,7 @@ func TestIssue1034(t *testing.T) {
 		assertNoError(p.Continue(), t, "Continue()")
 		frames, err := p.SelectedGoroutine().Stacktrace(10, 0)
 		assertNoError(err, t, "Stacktrace")
-		scope := proc.FrameToScope(p.BinInfo(), p.Memory(), nil, frames[2:]...)
+		scope := proc.FrameToScope(p, p.BinInfo(), p.Memory(), nil, frames[2:]...)
 		args, _ := scope.FunctionArguments(normalLoadConfig)
 		assertNoError(err, t, "FunctionArguments()")
 		if len(args) > 0 {
@@ -3446,7 +3446,7 @@ func testDeclLineCount(t *testing.T, p *proc.Target, lineno int, tgtvars []strin
 	sort.Strings(tgtvars)
 
 	assertLineNumber(p, t, lineno, "Program did not continue to correct next location")
-	scope, err := proc.GoroutineScope(p.CurrentThread())
+	scope, err := proc.GoroutineScope(p, p.CurrentThread())
 	assertNoError(err, t, fmt.Sprintf("GoroutineScope (:%d)", lineno))
 	vars, err := scope.Locals()
 	assertNoError(err, t, fmt.Sprintf("Locals (:%d)", lineno))
@@ -3864,7 +3864,7 @@ func TestIssue951(t *testing.T) {
 
 	withTestProcess("issue951", t, func(p *proc.Target, fixture protest.Fixture) {
 		assertNoError(p.Continue(), t, "Continue()")
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, t, "GoroutineScope")
 		args, err := scope.FunctionArguments(normalLoadConfig)
 		assertNoError(err, t, "FunctionArguments")
@@ -3910,7 +3910,7 @@ func TestMapLoadConfigWithReslice(t *testing.T) {
 	withTestProcess("testvariables2", t, func(p *proc.Target, fixture protest.Fixture) {
 		zolotovLoadCfg := proc.LoadConfig{FollowPointers: true, MaxStructFields: -1, MaxVariableRecurse: 3, MaxStringLen: 10, MaxArrayValues: 10}
 		assertNoError(p.Continue(), t, "First Continue()")
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, t, "GoroutineScope")
 		m1, err := scope.EvalExpression("m1", zolotovLoadCfg)
 		assertNoError(err, t, "EvalVariable")
@@ -4152,7 +4152,7 @@ func TestIssue1432(t *testing.T) {
 		svar := evalVariable(p, t, "s")
 		t.Logf("%#x", svar.Addr)
 
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, t, "GoroutineScope()")
 
 		err = scope.SetVariable(fmt.Sprintf("(*\"main.s\")(%#x).i", svar.Addr), "10")
@@ -5054,8 +5054,8 @@ func TestDump(t *testing.T) {
 					t.Errorf("Frame mismatch %d.%d\nlive:\t%s\ncore:\t%s", gos[i].ID, j, convertFrame(p.BinInfo().Arch, &frames[j]), convertFrame(p.BinInfo().Arch, &cframes[j]))
 				}
 				if frames[j].Call.Fn != nil && frames[j].Call.Fn.Name == "main.main" {
-					scope = proc.FrameToScope(p.BinInfo(), p.Memory(), gos[i], frames[j:]...)
+					scope = proc.FrameToScope(p, p.BinInfo(), p.Memory(), gos[i], frames[j:]...)
-					cscope = proc.FrameToScope(c.BinInfo(), c.Memory(), cgos[i], cframes[j:]...)
+					cscope = proc.FrameToScope(c, c.BinInfo(), c.Memory(), cgos[i], cframes[j:]...)
 				}
 			}
 		}
@@ -5132,9 +5132,11 @@ func TestCompositeMemoryWrite(t *testing.T) {
 			return regs.PC(), rax, xmm1
 		}
 
+		const fakeAddress = 0xbeef0000
+
 		getmem := func(mem proc.MemoryReader) uint64 {
 			buf := make([]byte, 8)
-			_, err := mem.ReadMemory(buf, 0xbeef0000)
+			_, err := mem.ReadMemory(buf, fakeAddress)
 			assertNoError(err, t, "ReadMemory")
 			return binary.LittleEndian.Uint64(buf)
 		}
@@ -5143,9 +5145,9 @@ func TestCompositeMemoryWrite(t *testing.T) {
 		oldPc, oldRax, oldXmm1 := getregs()
 		t.Logf("PC %#x AX %#x XMM1 %#x", oldPc, oldRax, oldXmm1)
 
-		memRax, err := proc.NewCompositeMemory(p, []op.Piece{{Size: 0, Val: 0, Kind: op.RegPiece}})
+		memRax, err := proc.NewCompositeMemory(p, []op.Piece{{Size: 0, Val: 0, Kind: op.RegPiece}}, fakeAddress)
 		assertNoError(err, t, "NewCompositeMemory (rax)")
-		memXmm1, err := proc.NewCompositeMemory(p, []op.Piece{{Size: 0, Val: 18, Kind: op.RegPiece}})
+		memXmm1, err := proc.NewCompositeMemory(p, []op.Piece{{Size: 0, Val: 18, Kind: op.RegPiece}}, fakeAddress)
 		assertNoError(err, t, "NewCompositeMemory (xmm1)")
 
 		if memRax := getmem(memRax); memRax != oldRax {
@@ -5204,7 +5206,7 @@ func TestWatchpointsBasic(t *testing.T) {
 		assertNoError(p.Continue(), t, "Continue 0")
 		assertLineNumber(p, t, 11, "Continue 0") // Position 0
 
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, t, "GoroutineScope")
 
 		bp, err := p.SetWatchpoint(scope, "globalvar1", proc.WatchWrite, nil)
@@ -5250,7 +5252,7 @@ func TestWatchpointCounts(t *testing.T) {
 		setFunctionBreakpoint(p, t, "main.main")
 		assertNoError(p.Continue(), t, "Continue 0")
 
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, t, "GoroutineScope")
 
 		bp, err := p.SetWatchpoint(scope, "globalvar1", proc.WatchWrite, nil)

pkg/proc/scope_test.go

@@ -238,7 +238,7 @@ func (check *scopeCheck) Parse(descr string, t *testing.T) {
 }
 
 func (scopeCheck *scopeCheck) checkLocalsAndArgs(p *proc.Target, t *testing.T) (*proc.EvalScope, bool) {
-	scope, err := proc.GoroutineScope(p.CurrentThread())
+	scope, err := proc.GoroutineScope(p, p.CurrentThread())
 	assertNoError(err, t, "GoroutineScope()")
 
 	ok := true

pkg/proc/stack.go

@@ -620,8 +620,8 @@ func (d *Defer) Next() *Defer {
 // EvalScope returns an EvalScope relative to the argument frame of this deferred call.
 // The argument frame of a deferred call is stored in memory immediately
 // after the deferred header.
-func (d *Defer) EvalScope(thread Thread) (*EvalScope, error) {
+func (d *Defer) EvalScope(t *Target, thread Thread) (*EvalScope, error) {
-	scope, err := GoroutineScope(thread)
+	scope, err := GoroutineScope(t, thread)
 	if err != nil {
 		return nil, fmt.Errorf("could not get scope: %v", err)
 	}

pkg/proc/target.go

@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"go/constant"
 	"os"
+	"sort"
 	"strings"
 
+	"github.com/go-delve/delve/pkg/dwarf/op"
 	"github.com/go-delve/delve/pkg/goversion"
 )
 
@@ -66,6 +68,12 @@ type Target struct {
 	// exitStatus is the exit status of the process we are debugging.
 	// Saved here to relay to any future commands.
 	exitStatus int
+
+	// fakeMemoryRegistry contains the list of all compositeMemory objects
+	// created since the last restart, it exists so that registerized variables
+	// can be given a unique address.
+	fakeMemoryRegistry    []*compositeMemory
+	fakeMemoryRegistryMap map[string]*compositeMemory
 }
 
 // ErrProcessExited indicates that the process has exited and contains both
@@ -181,6 +189,7 @@ func NewTarget(p Process, currentThread Thread, cfg NewTargetConfig) (*Target, e
 	t.createFatalThrowBreakpoint()
 
 	t.gcache.init(p.BinInfo())
+	t.fakeMemoryRegistryMap = make(map[string]*compositeMemory)
 
 	if cfg.DisableAsyncPreempt {
 		setAsyncPreemptOff(t, 1)
@@ -232,9 +241,10 @@ func (t *Target) SupportsFunctionCalls() bool {
 	return t.Process.BinInfo().Arch.Name == "amd64"
 }
 
-// ClearAllGCache clears the internal Goroutine cache.
+// ClearCaches clears internal caches that should not survive a restart.
 // This should be called anytime the target process executes instructions.
-func (t *Target) ClearAllGCache() {
+func (t *Target) ClearCaches() {
+	t.clearFakeMemory()
 	t.gcache.Clear()
 	for _, thread := range t.ThreadList() {
 		thread.Common().g = nil
@@ -245,7 +255,7 @@ func (t *Target) ClearAllGCache() {
 // This is only useful for recorded targets.
 // Restarting of a normal process happens at a higher level (debugger.Restart).
 func (t *Target) Restart(from string) error {
-	t.ClearAllGCache()
+	t.ClearCaches()
 	currentThread, err := t.proc.Restart(from)
 	if err != nil {
 		return err
@@ -385,3 +395,73 @@ func (t *Target) CurrentThread() Thread {
 func (t *Target) SetNextBreakpointID(id int) {
 	t.Breakpoints().breakpointIDCounter = id
 }
+
+const (
+	fakeAddressBase     = 0xbeef000000000000
+	fakeAddressUnresolv = 0xbeed000000000000 // this address never resloves to memory
+)
+
+// newCompositeMemory creates a new compositeMemory object and registers it.
+// If the same composite memory has been created before it will return a
+// cached object.
+// This caching is primarily done so that registerized variables don't get a
+// different address every time they are evaluated, which would be confusing
+// and leak memory.
+func (t *Target) newCompositeMemory(mem MemoryReadWriter, regs op.DwarfRegisters, pieces []op.Piece, descr *locationExpr) (int64, *compositeMemory, error) {
+	var key string
+	if regs.CFA != 0 && len(pieces) > 0 {
+		// key is created by concatenating the location expression with the CFA,
+		// this combination is guaranteed to be unique between resumes.
+		buf := new(strings.Builder)
+		fmt.Fprintf(buf, "%#x ", regs.CFA)
+		op.PrettyPrint(buf, descr.instr)
+		key = buf.String()
+
+		if cmem := t.fakeMemoryRegistryMap[key]; cmem != nil {
+			return int64(cmem.base), cmem, nil
+		}
+	}
+
+	cmem, err := newCompositeMemory(mem, t.BinInfo().Arch, regs, pieces)
+	if err != nil {
+		return 0, cmem, err
+	}
+	t.registerFakeMemory(cmem)
+	if key != "" {
+		t.fakeMemoryRegistryMap[key] = cmem
+	}
+	return int64(cmem.base), cmem, nil
+}
+
+func (t *Target) registerFakeMemory(mem *compositeMemory) (addr uint64) {
+	t.fakeMemoryRegistry = append(t.fakeMemoryRegistry, mem)
+	addr = fakeAddressBase
+	if len(t.fakeMemoryRegistry) > 1 {
+		prevMem := t.fakeMemoryRegistry[len(t.fakeMemoryRegistry)-2]
+		addr = uint64(alignAddr(int64(prevMem.base+uint64(len(prevMem.data))), 0x100)) // the call to alignAddr just makes the address look nicer, it is not necessary
+	}
+	mem.base = addr
+	return addr
+}
+
+func (t *Target) findFakeMemory(addr uint64) *compositeMemory {
+	i := sort.Search(len(t.fakeMemoryRegistry), func(i int) bool {
+		mem := t.fakeMemoryRegistry[i]
+		return addr <= mem.base || (mem.base <= addr && addr < (mem.base+uint64(len(mem.data))))
+	})
+	if i != len(t.fakeMemoryRegistry) {
+		mem := t.fakeMemoryRegistry[i]
+		if mem.base <= addr && addr < (mem.base+uint64(len(mem.data))) {
+			return mem
+		}
+	}
+	return nil
+}
+
+func (t *Target) clearFakeMemory() {
+	for i := range t.fakeMemoryRegistry {
+		t.fakeMemoryRegistry[i] = nil
+	}
+	t.fakeMemoryRegistry = t.fakeMemoryRegistry[:0]
+	t.fakeMemoryRegistryMap = make(map[string]*compositeMemory)
+}

pkg/proc/target_exec.go

@@ -68,7 +68,7 @@ func (dbp *Target) Continue() error {
 			dbp.ClearInternalBreakpoints()
 			return nil
 		}
-		dbp.ClearAllGCache()
+		dbp.ClearCaches()
 		trapthread, stopReason, err := dbp.proc.ContinueOnce()
 		dbp.StopReason = stopReason
 		if err != nil {
@@ -182,7 +182,7 @@ func (dbp *Target) Continue() error {
 					return dbp.StepInstruction()
 				}
 			default:
-				curthread.Common().returnValues = curbp.Breakpoint.returnInfo.Collect(curthread)
+				curthread.Common().returnValues = curbp.Breakpoint.returnInfo.Collect(dbp, curthread)
 				if err := dbp.ClearInternalBreakpoints(); err != nil {
 					return err
 				}
@@ -269,7 +269,7 @@ func disassembleCurrentInstruction(p Process, thread Thread, off int64) ([]AsmIn
 // This function is used to step out of runtime.Breakpoint as well as
 // runtime.debugCallV1.
 func stepInstructionOut(dbp *Target, curthread Thread, fnname1, fnname2 string) error {
-	defer dbp.ClearAllGCache()
+	defer dbp.ClearCaches()
 	for {
 		if err := curthread.StepInstruction(); err != nil {
 			return err
@@ -415,7 +415,7 @@ func (dbp *Target) StepInstruction() (err error) {
 		}
 		thread = g.Thread
 	}
-	dbp.ClearAllGCache()
+	dbp.ClearCaches()
 	if ok, err := dbp.Valid(); !ok {
 		return err
 	}

pkg/proc/variables.go

@@ -1125,7 +1125,7 @@ func readVarEntry(entry *godwarf.Tree, image *Image) (name string, typ godwarf.T
 
 // Extracts the name and type of a variable from a dwarf entry
 // then executes the instructions given in the  DW_AT_location attribute to grab the variable's address
-func extractVarInfoFromEntry(bi *BinaryInfo, image *Image, regs op.DwarfRegisters, mem MemoryReadWriter, entry *godwarf.Tree) (*Variable, error) {
+func extractVarInfoFromEntry(tgt *Target, bi *BinaryInfo, image *Image, regs op.DwarfRegisters, mem MemoryReadWriter, entry *godwarf.Tree) (*Variable, error) {
 	if entry.Tag != dwarf.TagFormalParameter && entry.Tag != dwarf.TagVariable {
 		return nil, fmt.Errorf("invalid entry tag, only supports FormalParameter and Variable, got %s", entry.Tag.String())
 	}
@@ -1137,9 +1137,16 @@ func extractVarInfoFromEntry(bi *BinaryInfo, image *Image, regs op.DwarfRegister
 
 	addr, pieces, descr, err := bi.Location(entry, dwarf.AttrLocation, regs.PC(), regs)
 	if pieces != nil {
-		addr = fakeAddress
 		var cmem *compositeMemory
-		cmem, err = newCompositeMemory(mem, bi.Arch, regs, pieces)
+		if tgt != nil {
+			addr, cmem, err = tgt.newCompositeMemory(mem, regs, pieces, descr)
+		} else {
+			cmem, err = newCompositeMemory(mem, bi.Arch, regs, pieces)
+			if cmem != nil {
+				cmem.base = fakeAddressUnresolv
+				addr = int64(cmem.base)
+			}
+		}
 		if cmem != nil {
 			mem = cmem
 		}
@@ -1246,7 +1253,7 @@ func (v *Variable) loadValueInternal(recurseLevel int, cfg LoadConfig) {
 
 		case v.Flags&VariableCPURegister != 0:
 			val = fmt.Sprintf("%x", v.reg.Bytes)
-			s := v.Base - fakeAddress
+			s := v.Base - fakeAddressUnresolv
 			if s < uint64(len(val)) {
 				val = val[s:]
 				if v.Len >= 0 && v.Len < int64(len(val)) {
@@ -2296,7 +2303,7 @@ func (v *Variable) registerVariableTypeConv(newtyp string) (*Variable, error) {
 	v.loaded = true
 	v.Kind = reflect.Array
 	v.Len = int64(len(v.Children))
-	v.Base = fakeAddress
+	v.Base = fakeAddressUnresolv
 	v.DwarfType = fakeArrayType(uint64(len(v.Children)), &godwarf.VoidType{CommonType: godwarf.CommonType{ByteSize: int64(n)}})
 	v.RealType = v.DwarfType
 	return v, nil

service/debugger/debugger.go

@@ -1293,7 +1293,7 @@ func (d *Debugger) collectBreakpointInformation(state *api.DebuggerState) error
 			continue
 		}
 
-		s, err := proc.GoroutineScope(thread)
+		s, err := proc.GoroutineScope(d.target, thread)
 		if err != nil {
 			return err
 		}
@@ -1398,7 +1398,7 @@ func (d *Debugger) PackageVariables(filter string, cfg proc.LoadConfig) ([]*proc
 		return nil, fmt.Errorf("invalid filter argument: %s", err.Error())
 	}
 
-	scope, err := proc.ThreadScope(d.target.CurrentThread())
+	scope, err := proc.ThreadScope(d.target, d.target.CurrentThread())
 	if err != nil {
 		return nil, err
 	}
@@ -1742,7 +1742,7 @@ func (d *Debugger) convertStacktrace(rawlocs []proc.Stackframe, cfg *proc.LoadCo
 		}
 		if cfg != nil && rawlocs[i].Current.Fn != nil {
 			var err error
-			scope := proc.FrameToScope(d.target.BinInfo(), d.target.Memory(), nil, rawlocs[i:]...)
+			scope := proc.FrameToScope(d.target, d.target.BinInfo(), d.target.Memory(), nil, rawlocs[i:]...)
 			locals, err := scope.LocalVariables(*cfg)
 			if err != nil {
 				return nil, err

service/test/integration2_test.go

@@ -2417,3 +2417,38 @@ func TestGoroutinesGrouping(t *testing.T) {
 		}
 	})
 }
+
+func TestLongStringArg(t *testing.T) {
+	// Test the ability to load more elements of a string argument, this could
+	// be broken if registerized variables are not handled correctly.
+	withTestClient2("morestringarg", t, func(c service.Client) {
+		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.f"})
+		assertNoError(err, t, "CreateBreakpoint")
+		state := <-c.Continue()
+		assertNoError(state.Err, t, "Continue")
+
+		test := func(name, val1, val2 string) uint64 {
+			var1, err := c.EvalVariable(api.EvalScope{GoroutineID: -1}, name, normalLoadConfig)
+			assertNoError(err, t, "EvalVariable")
+			t.Logf("%#v\n", var1)
+			if var1.Value != val1 {
+				t.Fatalf("wrong value for variable: %q", var1.Value)
+			}
+			var2, err := c.EvalVariable(api.EvalScope{GoroutineID: -1}, fmt.Sprintf("(*(*%q)(%#x))[64:]", var1.Type, var1.Addr), normalLoadConfig)
+			assertNoError(err, t, "EvalVariable")
+			t.Logf("%#v\n", var2)
+			if var2.Value != val2 {
+				t.Fatalf("wrong value for variable: %q", var2.Value)
+
+			}
+			return var1.Addr
+		}
+
+		saddr := test("s", "very long string 01234567890123456789012345678901234567890123456", "7890123456789012345678901234567890123456789X")
+		test("q", "very long string B 012345678901234567890123456789012345678901234", "567890123456789012345678901234567890123456789X2")
+		saddr2 := test("s", "very long string 01234567890123456789012345678901234567890123456", "7890123456789012345678901234567890123456789X")
+		if saddr != saddr2 {
+			t.Fatalf("address of s changed (%#x %#x)", saddr, saddr2)
+		}
+	})
+}

service/test/variables_test.go

@@ -86,13 +86,13 @@ func findFirstNonRuntimeFrame(p *proc.Target) (proc.Stackframe, error) {
 
 func evalScope(p *proc.Target) (*proc.EvalScope, error) {
 	if testBackend != "rr" {
-		return proc.GoroutineScope(p.CurrentThread())
+		return proc.GoroutineScope(p, p.CurrentThread())
 	}
 	frame, err := findFirstNonRuntimeFrame(p)
 	if err != nil {
 		return nil, err
 	}
-	return proc.FrameToScope(p.BinInfo(), p.Memory(), nil, frame), nil
+	return proc.FrameToScope(p, p.BinInfo(), p.Memory(), nil, frame), nil
 }
 
 func evalVariable(p *proc.Target, symbol string, cfg proc.LoadConfig) (*proc.Variable, error) {
@@ -111,7 +111,7 @@ func (tc *varTest) alternateVarTest() varTest {
 }
 
 func setVariable(p *proc.Target, symbol, value string) error {
-	scope, err := proc.GoroutineScope(p.CurrentThread())
+	scope, err := proc.GoroutineScope(p, p.CurrentThread())
 	if err != nil {
 		return err
 	}
@@ -471,10 +471,10 @@ func TestLocalVariables(t *testing.T) {
 				var frame proc.Stackframe
 				frame, err = findFirstNonRuntimeFrame(p)
 				if err == nil {
-					scope = proc.FrameToScope(p.BinInfo(), p.Memory(), nil, frame)
+					scope = proc.FrameToScope(p, p.BinInfo(), p.Memory(), nil, frame)
 				}
 			} else {
-				scope, err = proc.GoroutineScope(p.CurrentThread())
+				scope, err = proc.GoroutineScope(p, p.CurrentThread())
 			}
 
 			assertNoError(err, t, "scope")
@@ -1152,7 +1152,7 @@ func TestIssue1075(t *testing.T) {
 		setFunctionBreakpoint(p, t, "net/http.(*Client).Do")
 		assertNoError(p.Continue(), t, "Continue()")
 		for i := 0; i < 10; i++ {
-			scope, err := proc.GoroutineScope(p.CurrentThread())
+			scope, err := proc.GoroutineScope(p, p.CurrentThread())
 			assertNoError(err, t, fmt.Sprintf("GoroutineScope (%d)", i))
 			vars, err := scope.LocalVariables(pnormalLoadConfig)
 			assertNoError(err, t, fmt.Sprintf("LocalVariables (%d)", i))
@@ -1377,7 +1377,7 @@ func testCallFunction(t *testing.T, p *proc.Target, tc testCaseCallFunction) {
 	}
 
 	if varExpr != "" {
-		scope, err := proc.GoroutineScope(p.CurrentThread())
+		scope, err := proc.GoroutineScope(p, p.CurrentThread())
 		assertNoError(err, t, "GoroutineScope")
 		v, err := scope.EvalExpression(varExpr, pnormalLoadConfig)
 		assertNoError(err, t, fmt.Sprintf("EvalExpression(%s)", varExpr))