proc: fix prettyprint for register components with large values (#3022)

Fix pretty printing for CPU register components (created with the
XMM0.uintN syntax) while using format strings
Also fixes printing large literal constants with format strings.

Fixes #3020

_fixtures/xmm0print/main.go

@@ -0,0 +1,16 @@
+package main
+
+import "fmt"
+
+//go:noescape
+func VPSLLQ36(src, dst *[4]uint64)
+
+func main() {
+	src := [4]uint64{0: 0x38180a06, 1: 0x38180a06, 2: 0x18080200, 3: 0x18080200}
+	dst := [4]uint64{}
+	VPSLLQ36(&src, &dst)
+
+	for _, qword := range dst {
+		fmt.Printf("%064b\n", qword)
+	}
+}

_fixtures/xmm0print/main.s

@@ -0,0 +1,9 @@
+#include "textflag.h"
+
+TEXT ·VPSLLQ36(SB), NOSPLIT, $0-16
+    MOVQ src+0(FP), AX
+    MOVQ dst+8(FP), BX
+    VMOVDQU (AX), Y0
+    VPSLLQ $36, Y0, Y0
+    VMOVDQU Y0, (BX)
+    RET

pkg/proc/variables.go

@@ -740,11 +740,16 @@ func resolveTypedef(typ godwarf.Type) godwarf.Type {
 	}
 }
 
+var constantMaxInt64 = constant.MakeInt64(1<<63 - 1)
+
 func newConstant(val constant.Value, mem MemoryReadWriter) *Variable {
 	v := &Variable{Value: val, mem: mem, loaded: true}
 	switch val.Kind() {
 	case constant.Int:
 		v.Kind = reflect.Int
+		if constant.Sign(val) >= 0 && constant.Compare(val, token.GTR, constantMaxInt64) {
+			v.Kind = reflect.Uint64
+		}
 	case constant.Float:
 		v.Kind = reflect.Float64
 	case constant.Bool:
@@ -2278,37 +2283,47 @@ func (v *Variable) registerVariableTypeConv(newtyp string) (*Variable, error) {
 		switch newtyp {
 		case "int8":
 			child = newConstant(constant.MakeInt64(int64(int8(v.reg.Bytes[i]))), v.mem)
+			child.Kind = reflect.Int8
 			n = 1
 		case "int16":
 			child = newConstant(constant.MakeInt64(int64(int16(binary.LittleEndian.Uint16(v.reg.Bytes[i:])))), v.mem)
+			child.Kind = reflect.Int16
 			n = 2
 		case "int32":
 			child = newConstant(constant.MakeInt64(int64(int32(binary.LittleEndian.Uint32(v.reg.Bytes[i:])))), v.mem)
+			child.Kind = reflect.Int32
 			n = 4
 		case "int64":
 			child = newConstant(constant.MakeInt64(int64(binary.LittleEndian.Uint64(v.reg.Bytes[i:]))), v.mem)
+			child.Kind = reflect.Int64
 			n = 8
 		case "uint8":
 			child = newConstant(constant.MakeUint64(uint64(v.reg.Bytes[i])), v.mem)
+			child.Kind = reflect.Uint8
 			n = 1
 		case "uint16":
 			child = newConstant(constant.MakeUint64(uint64(binary.LittleEndian.Uint16(v.reg.Bytes[i:]))), v.mem)
+			child.Kind = reflect.Uint16
 			n = 2
 		case "uint32":
 			child = newConstant(constant.MakeUint64(uint64(binary.LittleEndian.Uint32(v.reg.Bytes[i:]))), v.mem)
+			child.Kind = reflect.Uint32
 			n = 4
 		case "uint64":
 			child = newConstant(constant.MakeUint64(uint64(binary.LittleEndian.Uint64(v.reg.Bytes[i:]))), v.mem)
+			child.Kind = reflect.Uint64
 			n = 8
 		case "float32":
 			a := binary.LittleEndian.Uint32(v.reg.Bytes[i:])
 			x := *(*float32)(unsafe.Pointer(&a))
 			child = newConstant(constant.MakeFloat64(float64(x)), v.mem)
+			child.Kind = reflect.Float32
 			n = 4
 		case "float64":
 			a := binary.LittleEndian.Uint64(v.reg.Bytes[i:])
 			x := *(*float64)(unsafe.Pointer(&a))
 			child = newConstant(constant.MakeFloat64(x), v.mem)
+			child.Kind = reflect.Float64
 			n = 8
 		default:
 			if n == 0 {

service/test/integration2_test.go

@@ -2767,3 +2767,53 @@ func TestRestartRewindAfterEnd(t *testing.T) {
 		}
 	})
 }
+
+func TestClientServer_SinglelineStringFormattedWithBigInts(t *testing.T) {
+	// Check that variables that represent large numbers are represented correctly when using a formatting string
+
+	if runtime.GOARCH != "amd64" {
+		t.Skip("N/A")
+	}
+	withTestClient2Extended("xmm0print/", t, 0, [3]string{}, func(c service.Client, fixture protest.Fixture) {
+		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.VPSLLQ36", Line: 4})
+		assertNoError(err, t, "CreateBreakpoint")
+		state := <-c.Continue()
+		if state.CurrentThread.Line != 8 {
+			t.Fatalf("wrong location after continue %s:%d", state.CurrentThread.File, state.CurrentThread.Line)
+		}
+
+		constvar, err := c.EvalVariable(api.EvalScope{GoroutineID: -1}, "9331634762088972288", normalLoadConfig)
+		assertNoError(err, t, "ErrVariable(9331634762088972288)")
+		out := constvar.SinglelineStringFormatted("%X")
+		t.Logf("constant: %q\n", out)
+		if out != "8180A06000000000" {
+			t.Errorf("expected \"8180A06000000000\" got %q when printing constant", out)
+		}
+
+		xmm0var, err := c.EvalVariable(api.EvalScope{GoroutineID: -1}, "XMM0.uint64", normalLoadConfig)
+		assertNoError(err, t, "EvalVariable(XMM0.uint64)")
+
+		expected := []string{
+			"9331634762088972288", "8180A06000000000",
+			"9331634762088972288", "8180A06000000000",
+			"9259436018245828608", "8080200000000000",
+			"9259436018245828608", "8080200000000000",
+		}
+
+		for i := range xmm0var.Children {
+			child := &xmm0var.Children[i]
+			if child.Kind != reflect.Uint64 {
+				t.Errorf("wrong kind for variable %s\n", child.Kind)
+			}
+			out1 := child.SinglelineString()
+			out2 := child.SinglelineStringFormatted("%X")
+			t.Logf("%q %q\n", out1, out2)
+			if out1 != expected[i*2] {
+				t.Errorf("for child %d expected %s got %s (decimal)", i, expected[i*2], out1)
+			}
+			if out2 != expected[i*2+1] {
+				t.Errorf("for child %d expected %s got %s (hexadecimal)", i, expected[i*2+1], out2)
+			}
+		}
+	})
+}