pkg/proc: Parse Goroutine ID in eBPF tracer (#2654)

This patch enables the eBPF tracer backend to parse the ID of the
Goroutine which hit the uprobe. This implementation is specific to AMD64
and will have to be generalized further in order to be used on other
architectures.

Makefile

@@ -10,6 +10,14 @@ check-cert:
 build: $(GO_SRC)
 	@go run _scripts/make.go build
 
+docker-image-build:
+	@docker build -t ebpf-builder:latest -f ./pkg/proc/internal/ebpf/trace_probe/Dockerfile ./pkg/proc/internal/ebpf/
+
+docker-ebpf-obj-build: docker-image-build
+	@docker run -it --rm \
+	-v $(abspath .):/delve \
+	ebpf-builder:latest
+
 $(BPF_OBJ): $(BPF_SRC)
 	clang \
 		-I /usr/include \
@@ -22,7 +30,7 @@ $(BPF_OBJ): $(BPF_SRC)
 		pkg/proc/internal/ebpf/trace_probe/trace.bpf.c
 
 build-bpf: $(BPF_OBJ) $(GO_SRC)
-	@env CGO_LDFLAGS="/usr/lib64/libbpf.a" go run _scripts/make.go build --tags=ebpf
+	@env CGO_LDFLAGS="/usr/lib/libbpf.a" go run _scripts/make.go build --tags=ebpf
 
 install: $(GO_SRC)
 	@go run _scripts/make.go install
@@ -45,4 +53,4 @@ test-integration-run:
 vendor:
 	@go run _scripts/make.go vendor
 
-.PHONY: vendor test-integration-run test-proc-run test check-cert install build vet build-bpf uninstall
+.PHONY: vendor test-integration-run test-proc-run test check-cert install build vet build-bpf uninstall docker-image-build docker-ebpf-obj-build

_scripts/test_linux.sh

@@ -4,6 +4,7 @@ set -x
 
 apt-get -qq update
 apt-get install -y dwz wget make git gcc curl jq lsof
+
 dwz --version
 
 version=$1
@@ -47,4 +48,5 @@ echo "$PATH"
 echo "$GOROOT"
 echo "$GOPATH"
 cd delve
+
 make test

_scripts/test_mac.sh

@@ -36,6 +36,8 @@ fi
 
 mkdir -p $TMPDIR/gopath
 
+go env
+
 export GOPATH="$TMPDIR/gopath"
 export GOARCH="$ARCH"
 export PATH="$GOROOT/bin:$PATH"

cmd/dlv/cmds/commands.go

@@ -657,7 +657,7 @@ func traceCmd(cmd *cobra.Command, args []string) {
 									params.WriteString(p.Value)
 								}
 							}
-							fmt.Printf("%s:%d %s(%s)\n", t.File, t.Line, t.FunctionName, params.String())
+							fmt.Fprintf(os.Stderr, "> (%d) %s(%s)\n", t.GoroutineID, t.FunctionName, params.String())
 						}
 					}
 				}

cmd/dlv/dlv_test.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -19,6 +20,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-delve/delve/pkg/goversion"
 	protest "github.com/go-delve/delve/pkg/proc/test"
 	"github.com/go-delve/delve/pkg/terminal"
 	"github.com/go-delve/delve/service/dap/daptest"
@@ -27,6 +29,11 @@ import (
 )
 
 var testBackend string
+var ldFlags string
+
+func init() {
+	ldFlags = os.Getenv("CGO_LDFLAGS")
+}
 
 func TestMain(m *testing.M) {
 	flag.StringVar(&testBackend, "backend", "", "selects backend")
@@ -188,13 +195,30 @@ func testOutput(t *testing.T, dlvbin, output string, delveCmds []string) (stdout
 }
 
 func getDlvBin(t *testing.T) (string, string) {
+	// In case this was set in the environment
+	// from getDlvBinEBPF lets clear it here so
+	// we can ensure we don't get build errors
+	// depending on the test ordering.
+	os.Setenv("CGO_LDFLAGS", ldFlags)
+	return getDlvBinInternal(t)
+}
+
+func getDlvBinEBPF(t *testing.T) (string, string) {
+	os.Setenv("CGO_LDFLAGS", "/usr/lib/libbpf.a")
+	return getDlvBinInternal(t, "-tags", "ebpf")
+}
+
+func getDlvBinInternal(t *testing.T, goflags ...string) (string, string) {
 	tmpdir, err := ioutil.TempDir("", "TestDlv")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	dlvbin := filepath.Join(tmpdir, "dlv.exe")
-	out, err := exec.Command("go", "build", "-o", dlvbin, "github.com/go-delve/delve/cmd/dlv").CombinedOutput()
+	args := append([]string{"build", "-o", dlvbin}, goflags...)
+	args = append(args, "github.com/go-delve/delve/cmd/dlv")
+
+	out, err := exec.Command("go", args...).CombinedOutput()
 	if err != nil {
 		t.Fatalf("go build -o %v github.com/go-delve/delve/cmd/dlv: %v\n%s", dlvbin, err, string(out))
 	}
@@ -765,6 +789,46 @@ func TestTracePrintStack(t *testing.T) {
 	}
 }
 
+func TestTraceEBPF(t *testing.T) {
+	if os.Getenv("CI") == "true" {
+		t.Skip("cannot run test in CI, requires kernel compiled with btf support")
+	}
+	if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
+		t.Skip("not implemented on non linux/amd64 systems")
+	}
+	if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 16) {
+		t.Skip("requires at least Go 1.16 to run test")
+	}
+	usr, err := user.Current()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if usr.Uid != "0" {
+		t.Skip("test must be run as root")
+	}
+
+	dlvbin, tmpdir := getDlvBinEBPF(t)
+	defer os.RemoveAll(tmpdir)
+
+	expected := []byte("> (1) main.foo(99, 9801)\n")
+
+	fixtures := protest.FindFixturesDir()
+	cmd := exec.Command(dlvbin, "trace", "--ebpf", "--output", filepath.Join(tmpdir, "__debug"), filepath.Join(fixtures, "issue573.go"), "foo")
+	rdr, err := cmd.StderrPipe()
+	assertNoError(err, t, "stderr pipe")
+	defer rdr.Close()
+
+	assertNoError(cmd.Start(), t, "running trace")
+
+	output, err := ioutil.ReadAll(rdr)
+	assertNoError(err, t, "ReadAll")
+
+	if !bytes.Contains(output, expected) {
+		t.Fatalf("expected:\n%s\ngot:\n%s", string(expected), string(output))
+	}
+	cmd.Wait()
+}
+
 func TestDlvTestChdir(t *testing.T) {
 	dlvbin, tmpdir := getDlvBin(t)
 	defer os.RemoveAll(tmpdir)

pkg/proc/breakpoints.go

@@ -10,6 +10,7 @@ import (
 	"go/token"
 	"reflect"
 
+	"github.com/go-delve/delve/pkg/dwarf/godwarf"
 	"github.com/go-delve/delve/pkg/dwarf/op"
 	"github.com/go-delve/delve/pkg/dwarf/reader"
 	"github.com/go-delve/delve/pkg/goversion"
@@ -463,15 +464,46 @@ func (t *Target) SetBreakpoint(addr uint64, kind BreakpointKind, cond ast.Expr)
 	return t.setBreakpointInternal(addr, kind, 0, cond)
 }
 
+// SetEBPFTracepoint will attach a uprobe to the function
+// specified by 'fnName'.
 func (t *Target) SetEBPFTracepoint(fnName string) error {
+	// Not every OS/arch that we support has support for eBPF,
+	// so check early and return an error if this is called on an
+	// unsupported system.
 	if !t.proc.SupportsBPF() {
 		return errors.New("eBPF is not supported")
 	}
+	// Start putting together the argument map. This will tell the eBPF program
+	// all of the arguments we want to trace and how to find them.
 	var args []ebpf.UProbeArgMap
 	fn, ok := t.BinInfo().LookupFunc[fnName]
 	if !ok {
 		return fmt.Errorf("could not find function %s", fnName)
 	}
+
+	// Get information on the Goroutine so we can tell the
+	// eBPF program where to find it in order to get the
+	// goroutine ID.
+	rdr := t.BinInfo().Images[0].DwarfReader()
+	rdr.SeekToTypeNamed("runtime.g")
+	typ, err := t.BinInfo().findType("runtime.g")
+	if err != nil {
+		return errors.New("could not find type for runtime.g")
+	}
+	var goidOffset int64
+	switch t := typ.(type) {
+	case *godwarf.StructType:
+		for _, field := range t.Field {
+			if field.Name == "goid" {
+				goidOffset = field.ByteOffset
+				break
+			}
+		}
+	}
+
+	// Start looping through each argument / return parameter for the function we
+	// are setting the uprobe on. Parse location information so that we can pass it
+	// along to the eBPF program.
 	dwarfTree, err := fn.cu.image.getDwarfTree(fn.offset)
 	if err != nil {
 		return err
@@ -505,7 +537,9 @@ func (t *Target) SetEBPFTracepoint(fnName string) error {
 		offset += int64(t.BinInfo().Arch.PtrSize())
 		args = append(args, ebpf.UProbeArgMap{Offset: offset, Size: dt.Size(), Kind: dt.Common().ReflectKind, Pieces: paramPieces, InReg: len(pieces) > 0})
 	}
-	t.proc.SetUProbe(fnName, args)
+
+	// Finally, set the uprobe on the function.
+	t.proc.SetUProbe(fnName, goidOffset, args)
 	return nil
 }
 

pkg/proc/core/core.go

@@ -274,7 +274,7 @@ func (p *process) SupportsBPF() bool {
 	return false
 }
 
-func (dbp *process) SetUProbe(fnName string, args []ebpf.UProbeArgMap) error {
+func (dbp *process) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProbeArgMap) error {
 	panic("not implemented")
 }
 

pkg/proc/gdbserial/gdbserver.go

@@ -370,7 +370,7 @@ func (dbp *gdbProcess) GetBufferedTracepoints() []ebpf.RawUProbeParams {
 	return nil
 }
 
-func (dbp *gdbProcess) SetUProbe(fnName string, args []ebpf.UProbeArgMap) error {
+func (dbp *gdbProcess) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProbeArgMap) error {
 	panic("not implemented")
 }
 

pkg/proc/interface.go

@@ -46,7 +46,7 @@ type ProcessInternal interface {
 	EraseBreakpoint(*Breakpoint) error
 
 	SupportsBPF() bool
-	SetUProbe(string, []ebpf.UProbeArgMap) error
+	SetUProbe(string, int64, []ebpf.UProbeArgMap) error
 
 	// DumpProcessNotes returns ELF core notes describing the process and its threads.
 	// Implementing this method is optional.

pkg/proc/internal/ebpf/context.go

@@ -27,5 +27,6 @@ type RawUProbeParam struct {
 
 type RawUProbeParams struct {
 	FnAddr      int
+	GoroutineID int
 	InputParams []*RawUProbeParam
 }

pkg/proc/internal/ebpf/helpers.go

@@ -51,11 +51,12 @@ func (ctx *EBPFContext) AttachUprobe(pid int, name string, offset uint32) error
 	return err
 }
 
-func (ctx *EBPFContext) UpdateArgMap(key uint64, args []UProbeArgMap) error {
+func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64) error {
 	if ctx.bpfArgMap == nil {
 		return errors.New("eBPF map not loaded")
 	}
-	params := createFunctionParameterList(key, args)
+	params := createFunctionParameterList(key, goidOffset, args)
+	params.g_addr_offset = C.longlong(gAddrOffset)
 	return ctx.bpfArgMap.Update(unsafe.Pointer(&key), unsafe.Pointer(&params))
 }
 
@@ -131,6 +132,7 @@ func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
 
 	var rawParams RawUProbeParams
 	rawParams.FnAddr = int(params.fn_addr)
+	rawParams.GoroutineID = int(params.goroutine_id)
 
 	for i := 0; i < int(params.n_parameters); i++ {
 		iparam := &RawUProbeParam{}
@@ -166,8 +168,9 @@ func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
 	return rawParams
 }
 
-func createFunctionParameterList(entry uint64, args []UProbeArgMap) C.function_parameter_list_t {
+func createFunctionParameterList(entry uint64, goidOffset int64, args []UProbeArgMap) C.function_parameter_list_t {
 	var params C.function_parameter_list_t
+	params.goid_offset = C.uint(goidOffset)
 	params.n_parameters = C.uint(len(args))
 	params.fn_addr = C.uint(entry)
 	for i, arg := range args {

pkg/proc/internal/ebpf/helpers_disabled.go

@@ -18,7 +18,7 @@ func (ctx *EBPFContext) AttachUprobe(pid int, name string, offset uint32) error
 	return errors.New("eBPF is disabled")
 }
 
-func (ctx *EBPFContext) UpdateArgMap(key uint64, args []UProbeArgMap) error {
+func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64) error {
 	return errors.New("eBPF is disabled")
 }
 

pkg/proc/internal/ebpf/trace_probe/Dockerfile

@@ -0,0 +1,6 @@
+FROM golang:1.16-alpine
+RUN apk --no-cache update && apk --no-cache add clang llvm make gcc libc6-compat coreutils linux-headers musl-dev elfutils-dev libelf-static zlib-static make libbpf-dev libbpf git
+
+WORKDIR /delve
+
+CMD [ "/usr/bin/make", "build-bpf" ]

pkg/proc/internal/ebpf/trace_probe/function_vals.bpf.h

@@ -26,8 +26,12 @@ typedef struct function_parameter {
 } function_parameter_t;
 
 // function_parameter_list holds info about the function parameters and
-// stores information on up to 8 parameters.
+// stores information on up to 6 parameters.
 typedef struct function_parameter_list {
+      unsigned int goid_offset; // Offset of the `goid` struct member.
+      long long g_addr_offset;  // Offset of the Goroutine struct from the TLS segment.
+      int goroutine_id;
+
       unsigned int fn_addr;
       unsigned int n_parameters;          // number of parameters.
       function_parameter_t params[6];     // list of parameters.

pkg/proc/internal/ebpf/trace_probe/trace.bpf.c

@@ -140,6 +140,42 @@ int parse_param(struct pt_regs *ctx, function_parameter_t *param) {
     return 0;
 }
 
+__always_inline
+int get_goroutine_id(function_parameter_list_t *parsed_args) {
+    // Since eBPF programs have such strict stack requirements
+    // me must implement our own heap using a ringbuffer.
+    // Reserve some memory in our "heap" for the task_struct.
+    struct task_struct *task;
+    task = bpf_ringbuf_reserve(&heap, sizeof(struct task_struct), 0); 
+    if (!task) {
+        return 0;
+    }
+
+    // Get the current task.
+    __u64 task_ptr = bpf_get_current_task();
+    if (!task_ptr)
+    {
+        bpf_ringbuf_discard(task, 0);
+        return 0;
+    }
+    // The bpf_get_current_task helper returns us the address of the task_struct in
+    // kernel memory. Use the bpf_probe_read_kernel helper to read the struct out of
+    // kernel memory.
+    bpf_probe_read_kernel(task, sizeof(struct task_struct), (void*)(task_ptr));
+
+    // Get the Goroutine ID which is stored in thread local storage.
+    __u64  goid;
+    size_t g_addr;
+    bpf_probe_read_user(&g_addr, sizeof(void *), (void*)(task->thread.fsbase+parsed_args->g_addr_offset));
+    bpf_probe_read_user(&goid, sizeof(void *), (void*)(g_addr+parsed_args->goid_offset));
+    parsed_args->goroutine_id = goid;
+
+    // Free back up the memory we reserved for the task_struct.
+    bpf_ringbuf_discard(task, 0);
+
+    return 1;
+}
+
 SEC("uprobe/dlv_trace")
 int uprobe__dlv_trace(struct pt_regs *ctx) {
     function_parameter_list_t *args;
@@ -158,6 +194,11 @@ int uprobe__dlv_trace(struct pt_regs *ctx) {
     }
     memcpy(parsed_args, args, sizeof(function_parameter_list_t));
 
+    if (!get_goroutine_id(parsed_args)) {
+        bpf_ringbuf_discard(parsed_args, 0);
+        return 1;
+    }
+
     // Since we cannot loop in eBPF programs let's take adavantage of the
     // fact that in C switch cases will pass through automatically.
     switch (args->n_parameters) {

pkg/proc/internal/ebpf/trace_probe/trace.bpf.h

@@ -11,6 +11,11 @@ struct {
    __uint(max_entries, BPF_MAX_VAR_SIZ);
 } events SEC(".maps");
 
+struct {
+   __uint(type, BPF_MAP_TYPE_RINGBUF);
+   __uint(max_entries, BPF_MAX_VAR_SIZ);
+} heap SEC(".maps");
+
 struct {
     __uint(max_entries, 42);
     __uint(type, BPF_MAP_TYPE_HASH);

pkg/proc/native/nonative_darwin.go

@@ -90,7 +90,7 @@ func (dbp *nativeProcess) SupportsBPF() bool {
 	panic(ErrNativeBackendDisabled)
 }
 
-func (dbp *nativeProcess) SetUProbe(fnName string, args []ebpf.UProbeArgMap) error {
+func (dbp *nativeProcess) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProbeArgMap) error {
 	panic(ErrNativeBackendDisabled)
 }
 

pkg/proc/native/proc_darwin.go

@@ -477,7 +477,7 @@ func (dbp *nativeProcess) SupportsBPF() bool {
 	return false
 }
 
-func (dbp *nativeProcess) SetUProbe(fnName string, args []ebpf.UProbeArgMap) error {
+func (dbp *nativeProcess) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProbeArgMap) error {
 	panic("not implemented")
 }
 

pkg/proc/native/proc_freebsd.go

@@ -381,7 +381,7 @@ func (dbp *nativeProcess) SupportsBPF() bool {
 	return false
 }
 
-func (dbp *nativeProcess) SetUProbe(fnName string, args []ebpf.UProbeArgMap) error {
+func (dbp *nativeProcess) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProbeArgMap) error {
 	panic("not implemented")
 }
 

pkg/proc/native/proc_linux.go

@@ -705,7 +705,7 @@ func (dbp *nativeProcess) EntryPoint() (uint64, error) {
 	return linutil.EntryPointFromAuxv(auxvbuf, dbp.bi.Arch.PtrSize()), nil
 }
 
-func (dbp *nativeProcess) SetUProbe(fnName string, args []ebpf.UProbeArgMap) error {
+func (dbp *nativeProcess) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProbeArgMap) error {
 	// Lazily load and initialize the BPF program upon request to set a uprobe.
 	if dbp.os.ebpf == nil {
 		dbp.os.ebpf, _ = ebpf.LoadEBPFTracingProgram()
@@ -717,22 +717,23 @@ func (dbp *nativeProcess) SetUProbe(fnName string, args []ebpf.UProbeArgMap) err
 		return errors.New("too many arguments in traced function, max is 6")
 	}
 
-	debugname := dbp.bi.Images[0].Path
-	offset, err := ebpf.SymbolToOffset(debugname, fnName)
-	if err != nil {
-		return err
-	}
-	err = dbp.os.ebpf.AttachUprobe(dbp.Pid(), debugname, offset)
-	if err != nil {
-		return err
-	}
 	fn, ok := dbp.bi.LookupFunc[fnName]
 	if !ok {
 		return fmt.Errorf("could not find function: %s", fnName)
 	}
 
 	key := fn.Entry
-	return dbp.os.ebpf.UpdateArgMap(key, args)
+	err := dbp.os.ebpf.UpdateArgMap(key, goidOffset, args, dbp.BinInfo().GStructOffset())
+	if err != nil {
+		return err
+	}
+
+	debugname := dbp.bi.Images[0].Path
+	offset, err := ebpf.SymbolToOffset(debugname, fnName)
+	if err != nil {
+		return err
+	}
+	return dbp.os.ebpf.AttachUprobe(dbp.Pid(), debugname, offset)
 }
 
 func killProcess(pid int) error {

pkg/proc/native/proc_windows.go

@@ -524,7 +524,7 @@ func (dbp *nativeProcess) SupportsBPF() bool {
 	return false
 }
 
-func (dbp *nativeProcess) SetUProbe(fnName string, args []ebpf.UProbeArgMap) error {
+func (dbp *nativeProcess) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProbeArgMap) error {
 	return nil
 }
 

pkg/proc/target.go

@@ -405,6 +405,7 @@ func (t *Target) CurrentThread() Thread {
 
 type UProbeTraceResult struct {
 	FnAddr      int
+	GoroutineID int
 	InputParams []*Variable
 }
 
@@ -414,6 +415,7 @@ func (t *Target) GetBufferedTracepoints() []*UProbeTraceResult {
 	for _, tp := range tracepoints {
 		r := &UProbeTraceResult{}
 		r.FnAddr = tp.FnAddr
+		r.GoroutineID = tp.GoroutineID
 		for _, ip := range tp.InputParams {
 			v := &Variable{}
 			v.RealType = ip.RealType

pkg/proc/variables.go

@@ -843,10 +843,11 @@ func (v *Variable) parseG() (*G, error) {
 		}
 		return nil, ErrNoGoroutine{tid: id}
 	}
-	for {
-		if _, isptr := v.RealType.(*godwarf.PtrType); !isptr {
-			break
-		}
+	isptr := func(t godwarf.Type) bool {
+		_, ok := t.(*godwarf.PtrType)
+		return ok
+	}
+	for isptr(v.RealType) {
 		v = v.maybeDereference() // +rtype g
 	}
 

service/api/types.go

@@ -52,7 +52,7 @@ type DebuggerState struct {
 }
 
 type TracepointResult struct {
-	// Addr is deprecated, use Addrs.
+	// Addr is the address of this tracepoint.
 	Addr uint64 `json:"addr"`
 	// File is the source file for the breakpoint.
 	File string `json:"file"`
@@ -62,6 +62,8 @@ type TracepointResult struct {
 	// may not always be available.
 	FunctionName string `json:"functionName,omitempty"`
 
+	GoroutineID int `json:"goroutineID"`
+
 	InputParams  []Variable `json:"inputParams,omitempty"`
 	ReturnParams []Variable `json:"returnParams,omitempty"`
 }

service/debugger/debugger.go

@@ -2150,6 +2150,7 @@ func (d *Debugger) GetBufferedTracepoints() []api.TracepointResult {
 		results[i].FunctionName = fn.Name
 		results[i].Line = l
 		results[i].File = f
+		results[i].GoroutineID = trace.GoroutineID
 
 		for _, p := range trace.InputParams {
 			results[i].InputParams = append(results[i].InputParams, *api.ConvertVar(p))