proc/gdbserial: workaround for debugserver register set bug (#2770)

Debugserver has a bug where writing to a AVX-2 or AVX-512 register does
not work unless it is followed by at least a write to a AVX (not 2 or
512) register.

See also: https://bugs.llvm.org/show_bug.cgi?id=52362

Fixes #2767

_fixtures/setymmreg/main.go

@@ -0,0 +1,7 @@
+package main
+
+func asmFunc()
+
+func main() {
+	asmFunc()
+}

_fixtures/setymmreg/setymmreg_amd64.s

@@ -0,0 +1,9 @@
+#include "textflag.h"
+
+TEXT ·asmFunc(SB),0,$0-0
+	XORQ AX, AX
+	XORQ AX, AX
+	XORQ AX, AX
+	XORQ AX, AX
+	XORQ AX, AX
+	RET

pkg/proc/gdbserial/gdbserver.go

@@ -1617,10 +1617,14 @@ func (regs *gdbRegisters) init(regsInfo []gdbRegisterInfo, arch *proc.Arch, regn
 	}
 	regs.buf = make([]byte, regsz)
 	for _, reginfo := range regsInfo {
-		regs.regs[reginfo.Name] = gdbRegister{regnum: reginfo.Regnum, value: regs.buf[reginfo.Offset : reginfo.Offset+reginfo.Bitsize/8], ignoreOnWrite: reginfo.ignoreOnWrite}
+		regs.regs[reginfo.Name] = regs.gdbRegisterNew(&reginfo)
 	}
 }
 
+func (regs *gdbRegisters) gdbRegisterNew(reginfo *gdbRegisterInfo) gdbRegister {
+	return gdbRegister{regnum: reginfo.Regnum, value: regs.buf[reginfo.Offset : reginfo.Offset+reginfo.Bitsize/8], ignoreOnWrite: reginfo.ignoreOnWrite}
+}
+
 // reloadRegisters loads the current value of the thread's registers.
 // It will also load the address of the thread's G.
 // Loading the address of G can be done in one of two ways reloadGAlloc, if
@@ -1963,7 +1967,21 @@ func (t *gdbThread) SetReg(regNum uint64, reg *op.DwarfRegister) error {
 		return fmt.Errorf("could not set register %s: wrong size, expected %d got %d", regName, len(gdbreg.value), len(reg.Bytes))
 	}
 	copy(gdbreg.value, reg.Bytes)
-	return t.p.conn.writeRegister(t.strID, gdbreg.regnum, gdbreg.value)
+	err := t.p.conn.writeRegister(t.strID, gdbreg.regnum, gdbreg.value)
+	if err != nil {
+		return err
+	}
+	if t.p.conn.workaroundReg != nil && len(gdbreg.value) > 16 {
+		// This is a workaround for a bug in debugserver where register writes (P
+		// packet) on AVX-2 and AVX-512 registers are ignored unless they are
+		// followed by a write to an AVX register.
+		// See:
+		//  Issue #2767
+		//  https://bugs.llvm.org/show_bug.cgi?id=52362
+		reg := t.regs.gdbRegisterNew(t.p.conn.workaroundReg)
+		return t.p.conn.writeRegister(t.strID, reg.regnum, reg.value)
+	}
+	return nil
 }
 
 func (regs *gdbRegisters) Slice(floatingPoint bool) ([]proc.Register, error) {

pkg/proc/gdbserial/gdbserver_conn.go

@@ -37,6 +37,8 @@ type gdbConn struct {
 	packetSize int               // maximum packet size supported by stub
 	regsInfo   []gdbRegisterInfo // list of registers
 
+	workaroundReg *gdbRegisterInfo // used to work-around a register setting bug in debugserver, see use in gdbserver.go
+
 	pid int // cache process id
 
 	ack                   bool // when ack is true acknowledgment packets are enabled
@@ -338,6 +340,9 @@ func (conn *gdbConn) readRegisterInfo(regFound map[string]bool) (err error) {
 		}
 
 		if contained {
+			if regname == "xmm0" {
+				conn.workaroundReg = &gdbRegisterInfo{Regnum: regnum, Name: regname, Bitsize: bitsize, Offset: offset, ignoreOnWrite: ignoreOnWrite}
+			}
 			regnum++
 			continue
 		}

pkg/proc/proc_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/go-delve/delve/pkg/dwarf/frame"
 	"github.com/go-delve/delve/pkg/dwarf/op"
+	"github.com/go-delve/delve/pkg/dwarf/regnum"
 	"github.com/go-delve/delve/pkg/goversion"
 	"github.com/go-delve/delve/pkg/logflags"
 	"github.com/go-delve/delve/pkg/proc"
@@ -5696,3 +5697,42 @@ func TestSetOnFunctions(t *testing.T) {
 		}
 	})
 }
+
+func TestSetYMMRegister(t *testing.T) {
+	skipUnlessOn(t, "N/A", "darwin", "amd64")
+	// Checks that setting a XMM register works. This checks that the
+	// workaround for a bug in debugserver works.
+	// See issue #2767.
+	withTestProcess("setymmreg/", t, func(p *proc.Target, fixture protest.Fixture) {
+		setFunctionBreakpoint(p, t, "main.asmFunc")
+		assertNoError(p.Continue(), t, "Continue()")
+
+		getReg := func(pos string) *op.DwarfRegister {
+			regs := getRegisters(p, t)
+
+			arch := p.BinInfo().Arch
+			dregs := arch.RegistersToDwarfRegisters(0, regs)
+
+			r := dregs.Reg(regnum.AMD64_XMM0)
+			t.Logf("%s: %#v", pos, r)
+			return r
+		}
+
+		getReg("before")
+
+		p.CurrentThread().SetReg(regnum.AMD64_XMM0, op.DwarfRegisterFromBytes([]byte{
+			0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+			0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+			0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+			0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44}))
+		assertNoError(p.CurrentThread().StepInstruction(), t, "SetpInstruction")
+
+		xmm0 := getReg("after")
+
+		for i := range xmm0.Bytes {
+			if xmm0.Bytes[i] != 0x44 {
+				t.Fatalf("wrong register value")
+			}
+		}
+	})
+}