PenaDevops/delve
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

eBPF tracing backend return value parsing (#2704)

Browse Source 
Add return value parsing for eBPF tracing backend.
This commit is contained in:
Derek Parker 2021-10-25 12:37:36 -07:00 committed by GitHub
parent 8ebd2d83ae
commit 689e08260b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 234 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/dlv/cmds/commands.go
View File

@ -673,6 +673,7 @@ func traceCmd(cmd *cobra.Command, args []string) {
done := make(chan struct{})
defer close(done)
go func() {
gFnEntrySeen := map[int]struct{}{}
for {
select {
case <-done:
@ -694,7 +695,16 @@ func traceCmd(cmd *cobra.Command, args []string) {
params.WriteString(p.Value)
}
}
fmt.Fprintf(os.Stderr, "> (%d) %s(%s)\n", t.GoroutineID, t.FunctionName, params.String())
_, seen := gFnEntrySeen[t.GoroutineID]
if seen {
for _, p := range t.ReturnParams {
fmt.Fprintf(os.Stderr, "=> %#v\n", p.Value)
}
delete(gFnEntrySeen, t.GoroutineID)
} else {
gFnEntrySeen[t.GoroutineID] = struct{}{}
fmt.Fprintf(os.Stderr, "> (%d) %s(%s)\n", t.GoroutineID, t.FunctionName, params.String())
}
}
}
}

2
cmd/dlv/dlv_test.go
View File

@ -871,7 +871,7 @@ func TestTraceEBPF(t *testing.T) {
dlvbin, tmpdir := getDlvBinEBPF(t)
defer os.RemoveAll(tmpdir)
expected := []byte("> (1) main.foo(99, 9801)\n")
expected := []byte("> (1) main.foo(99, 9801)\n=> \"9900\"")
fixtures := protest.FindFixturesDir()
cmd := exec.Command(dlvbin, "trace", "--ebpf", "--output", filepath.Join(tmpdir, "__debug"), filepath.Join(fixtures, "issue573.go"), "foo")

17
pkg/proc/breakpoints.go
View File

@ -494,7 +494,6 @@ func (t *Target) SetEBPFTracepoint(fnName string) error {
}
// Start putting together the argument map. This will tell the eBPF program
// all of the arguments we want to trace and how to find them.
var args []ebpf.UProbeArgMap
fn, ok := t.BinInfo().LookupFunc[fnName]
if !ok {
return fmt.Errorf("could not find function %s", fnName)
@ -533,16 +532,14 @@ func (t *Target) SetEBPFTracepoint(fnName string) error {
}
_, l, _ := t.BinInfo().PCToLine(fn.Entry)
var args []ebpf.UProbeArgMap
varEntries := reader.Variables(dwarfTree, fn.Entry, l, variablesFlags)
for _, entry := range varEntries {
isret, _ := entry.Val(dwarf.AttrVarParam).(bool)
if isret {
continue
}
_, dt, err := readVarEntry(entry.Tree, fn.cu.image)
if err != nil {
return err
}
offset, pieces, _, err := t.BinInfo().Location(entry, dwarf.AttrLocation, fn.Entry, op.DwarfRegisters{}, nil)
if err != nil {
return err
@ -553,8 +550,16 @@ func (t *Target) SetEBPFTracepoint(fnName string) error {
paramPieces = append(paramPieces, int(piece.Val))
}
}
isret, _ := entry.Val(dwarf.AttrVarParam).(bool)
offset += int64(t.BinInfo().Arch.PtrSize())
args = append(args, ebpf.UProbeArgMap{Offset: offset, Size: dt.Size(), Kind: dt.Common().ReflectKind, Pieces: paramPieces, InReg: len(pieces) > 0})
args = append(args, ebpf.UProbeArgMap{
Offset: offset,
Size: dt.Size(),
Kind: dt.Common().ReflectKind,
Pieces: paramPieces,
InReg: len(pieces) > 0,
Ret: isret,
})
}
// Finally, set the uprobe on the function.

8
pkg/proc/core/core.go
View File

@ -281,6 +281,14 @@ func (dbp *process) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProb
// StartCallInjection notifies the backend that we are about to inject a function call.
func (p *process) StartCallInjection() (func(), error) { return func() {}, nil }
func (dbp *process) EnableURetProbes() error {
panic("not implemented")
}
func (dbp *process) DisableURetProbes() error {
panic("not implemented")
}
// ReadMemory will return memory from the core file at the specified location and put the
// read memory into `data`, returning the length read, and returning an error if
// the length read is shorter than the length of the `data` buffer.

8
pkg/proc/internal/ebpf/context.go
View File

@ -13,6 +13,7 @@ type UProbeArgMap struct {
Kind reflect.Kind // Kind of variable.
Pieces []int // Pieces of the variables as stored in registers.
InReg bool // True if this param is contained in a register.
Ret bool // True if this param is a return value.
}
type RawUProbeParam struct {
@ -26,7 +27,8 @@ type RawUProbeParam struct {
}
type RawUProbeParams struct {
FnAddr int
GoroutineID int
InputParams []*RawUProbeParam
FnAddr int
GoroutineID int
InputParams []*RawUProbeParam
ReturnParams []*RawUProbeParam
}

68
pkg/proc/internal/ebpf/helpers.go
View File

@ -6,6 +6,7 @@ package ebpf
// #include "./trace_probe/function_vals.bpf.h"
import "C"
import (
"debug/elf"
_ "embed"
"encoding/binary"
"errors"
@ -51,11 +52,11 @@ func (ctx *EBPFContext) AttachUprobe(pid int, name string, offset uint32) error
return err
}
func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64) error {
func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64, isret bool) error {
if ctx.bpfArgMap == nil {
return errors.New("eBPF map not loaded")
}
params := createFunctionParameterList(key, goidOffset, args)
params := createFunctionParameterList(key, goidOffset, args, isret)
params.g_addr_offset = C.longlong(gAddrOffset)
return ctx.bpfArgMap.Update(unsafe.Pointer(&key), unsafe.Pointer(&params))
}
@ -82,7 +83,7 @@ func LoadEBPFTracingProgram() (*EBPFContext, error) {
var ctx EBPFContext
var err error
ctx.bpfModule, err = bpf.NewModuleFromBuffer(TraceProbeBytes, "trace.o")
ctx.bpfModule, err = bpf.NewModuleFromBuffer(TraceProbeBytes, "trace_probe/trace.o")
if err != nil {
return nil, err
}
@ -114,7 +115,7 @@ func LoadEBPFTracingProgram() (*EBPFContext, error) {
return
}
parsed := ParseFunctionParameterList(b)
parsed := parseFunctionParameterList(b)
ctx.m.Lock()
ctx.parsedBpfEvents = append(ctx.parsedBpfEvents, parsed)
@ -125,7 +126,7 @@ func LoadEBPFTracingProgram() (*EBPFContext, error) {
return &ctx, nil
}
func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
func parseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
params := (*C.function_parameter_list_t)(unsafe.Pointer(&rawParamBytes[0]))
defer runtime.KeepAlive(params) // Ensure the param is not garbage collected.
@ -134,10 +135,10 @@ func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
rawParams.FnAddr = int(params.fn_addr)
rawParams.GoroutineID = int(params.goroutine_id)
for i := 0; i < int(params.n_parameters); i++ {
parseParam := func(param C.function_parameter_t) *RawUProbeParam {
iparam := &RawUProbeParam{}
data := make([]byte, 0x60)
ret := params.params[i]
ret := param
iparam.Kind = reflect.Kind(ret.kind)
val := C.GoBytes(unsafe.Pointer(&ret.val), C.int(ret.size))
@ -161,22 +162,30 @@ func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
iparam.Base = FakeAddressBase + 0x30
iparam.Len = int64(strLen)
}
return iparam
}
rawParams.InputParams = append(rawParams.InputParams, iparam)
for i := 0; i < int(params.n_parameters); i++ {
rawParams.InputParams = append(rawParams.InputParams, parseParam(params.params[i]))
}
for i := 0; i < int(params.n_ret_parameters); i++ {
rawParams.ReturnParams = append(rawParams.ReturnParams, parseParam(params.ret_params[i]))
}
return rawParams
}
func createFunctionParameterList(entry uint64, goidOffset int64, args []UProbeArgMap) C.function_parameter_list_t {
func createFunctionParameterList(entry uint64, goidOffset int64, args []UProbeArgMap, isret bool) C.function_parameter_list_t {
var params C.function_parameter_list_t
params.goid_offset = C.uint(goidOffset)
params.n_parameters = C.uint(len(args))
params.fn_addr = C.uint(entry)
for i, arg := range args {
params.is_ret = C.bool(isret)
params.n_parameters = C.uint(0)
params.n_ret_parameters = C.uint(0)
for _, arg := range args {
var param C.function_parameter_t
param.size = C.uint(arg.Size)
param.offset = C.uint(arg.Offset)
param.offset = C.int(arg.Offset)
param.kind = C.uint(arg.Kind)
if arg.InReg {
param.in_reg = true
@ -188,7 +197,40 @@ func createFunctionParameterList(entry uint64, goidOffset int64, args []UProbeAr
param.reg_nums[i] = C.int(arg.Pieces[i])
}
}
params.params[i] = param
if !arg.Ret {
params.params[params.n_parameters] = param
params.n_parameters++
} else {
params.ret_params[params.n_ret_parameters] = param
params.n_ret_parameters++
}
}
return params
}
func AddressToOffset(f *elf.File, addr uint64) (uint32, error) {
sectionsToSearchForSymbol := []*elf.Section{}
for i := range f.Sections {
if f.Sections[i].Flags == elf.SHF_ALLOC+elf.SHF_EXECINSTR {
sectionsToSearchForSymbol = append(sectionsToSearchForSymbol, f.Sections[i])
}
}
var executableSection *elf.Section
// Find what section the symbol is in by checking the executable section's
// addr space.
for m := range sectionsToSearchForSymbol {
if addr > sectionsToSearchForSymbol[m].Addr &&
addr < sectionsToSearchForSymbol[m].Addr+sectionsToSearchForSymbol[m].Size {
executableSection = sectionsToSearchForSymbol[m]
}
}
if executableSection == nil {
return 0, errors.New("could not find symbol in executable sections of binary")
}
return uint32(addr - executableSection.Addr + executableSection.Offset), nil
}

11
pkg/proc/internal/ebpf/helpers_disabled.go
View File

@ -4,6 +4,7 @@
package ebpf
import (
"debug/elf"
"errors"
)
@ -18,7 +19,11 @@ func (ctx *EBPFContext) AttachUprobe(pid int, name string, offset uint32) error
return errors.New("eBPF is disabled")
}
func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64) error {
func (ctx *EBPFContext) AttachURetprobe(pid int, name string, offset uint32) error {
return errors.New("eBPF is disabled")
}
func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64, isret bool) error {
return errors.New("eBPF is disabled")
}
@ -34,6 +39,6 @@ func LoadEBPFTracingProgram() (*EBPFContext, error) {
return nil, errors.New("eBPF disabled")
}
func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
return RawUProbeParams{}
func AddressToOffset(f *elf.File, addr uint64) (uint32, error) {
return 0, errors.New("eBPF disabled")
}

9
pkg/proc/internal/ebpf/trace_probe/function_vals.bpf.h
View File

@ -8,7 +8,7 @@ typedef struct function_parameter {
unsigned int size;
// Offset from stack pointer. This should only be set from the Go side.
unsigned int offset;
int offset;
// If true, the parameter is passed in a register.
bool in_reg;
@ -20,7 +20,7 @@ typedef struct function_parameter {
int reg_nums[6];
// The following are filled in by the eBPF program.
unsigned int daddr; // Data address.
size_t daddr; // Data address.
char val[0x30]; // Value of the parameter.
char deref_val[0x30]; // Dereference value of the parameter.
} function_parameter_t;
@ -33,6 +33,11 @@ typedef struct function_parameter_list {
int goroutine_id;
unsigned int fn_addr;
bool is_ret;
unsigned int n_parameters; // number of parameters.
function_parameter_t params[6]; // list of parameters.
unsigned int n_ret_parameters; // number of return parameters.
function_parameter_t ret_params[6]; // list of return parameters.
} function_parameter_list_t;

61
pkg/proc/internal/ebpf/trace_probe/trace.bpf.c
View File

@ -124,9 +124,9 @@ int parse_param(struct pt_regs *ctx, function_parameter_t *param) {
// a slice we will need some further processing below.
int ret = 0;
if (param->in_reg) {
parse_param_registers(ctx, param);
ret = parse_param_registers(ctx, param);
} else {
parse_param_stack(ctx, param);
ret = parse_param_stack(ctx, param);
}
if (ret != 0) {
return ret;
@ -176,11 +176,30 @@ int get_goroutine_id(function_parameter_list_t *parsed_args) {
return 1;
}
__always_inline
void parse_params(struct pt_regs *ctx, unsigned int n_params, function_parameter_t params[6]) {
// Since we cannot loop in eBPF programs let's take adavantage of the
// fact that in C switch cases will pass through automatically.
switch (n_params) {
case 6:
parse_param(ctx, &params[5]);
case 5:
parse_param(ctx, &params[4]);
case 4:
parse_param(ctx, &params[3]);
case 3:
parse_param(ctx, &params[2]);
case 2:
parse_param(ctx, &params[1]);
case 1:
parse_param(ctx, &params[0]);
}
}
SEC("uprobe/dlv_trace")
int uprobe__dlv_trace(struct pt_regs *ctx) {
function_parameter_list_t *args;
function_parameter_list_t *parsed_args;
function_parameter_t param;
uint64_t key = ctx->ip;
args = bpf_map_lookup_elem(&arg_map, &key);
@ -192,28 +211,32 @@ int uprobe__dlv_trace(struct pt_regs *ctx) {
if (!parsed_args) {
return 1;
}
memcpy(parsed_args, args, sizeof(function_parameter_list_t));
// Initialize the parsed_args struct.
parsed_args->goid_offset = args->goid_offset;
parsed_args->g_addr_offset = args->g_addr_offset;
parsed_args->goroutine_id = args->goroutine_id;
parsed_args->fn_addr = args->fn_addr;
parsed_args->n_parameters = args->n_parameters;
parsed_args->n_ret_parameters = args->n_ret_parameters;
memcpy(parsed_args->params, args->params, sizeof(args->params));
memcpy(parsed_args->ret_params, args->ret_params, sizeof(args->ret_params));
if (!get_goroutine_id(parsed_args)) {
bpf_ringbuf_discard(parsed_args, 0);
return 1;
}
// Since we cannot loop in eBPF programs let's take adavantage of the
// fact that in C switch cases will pass through automatically.
switch (args->n_parameters) {
case 6:
parse_param(ctx, &parsed_args->params[5]);
case 5:
parse_param(ctx, &parsed_args->params[4]);
case 4:
parse_param(ctx, &parsed_args->params[3]);
case 3:
parse_param(ctx, &parsed_args->params[2]);
case 2:
parse_param(ctx, &parsed_args->params[1]);
case 1:
parse_param(ctx, &parsed_args->params[0]);
if (!args->is_ret) {
// In uprobe at function entry.
// Parse input parameters.
parse_params(ctx, args->n_parameters, parsed_args->params);
} else {
// We are now stopped at the RET instruction for this function.
// Parse output parameters.
parse_params(ctx, args->n_ret_parameters, parsed_args->ret_params);
}
bpf_ringbuf_submit(parsed_args, 0);

1
pkg/proc/internal/ebpf/trace_probe/trace.bpf.h
View File

@ -16,6 +16,7 @@ struct {
__uint(max_entries, BPF_MAX_VAR_SIZ);
} heap SEC(".maps");
// Map which uses instruction address as key and function parameter info as the value.
struct {
__uint(max_entries, 42);
__uint(type, BPF_MAP_TYPE_HASH);

59
pkg/proc/native/proc_linux.go
View File

@ -3,6 +3,7 @@ package native
import (
"bufio"
"bytes"
"debug/elf"
"errors"
"fmt"
"io/ioutil"
@ -708,13 +709,18 @@ func (dbp *nativeProcess) EntryPoint() (uint64, error) {
func (dbp *nativeProcess) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProbeArgMap) error {
// Lazily load and initialize the BPF program upon request to set a uprobe.
if dbp.os.ebpf == nil {
dbp.os.ebpf, _ = ebpf.LoadEBPFTracingProgram()
var err error
dbp.os.ebpf, err = ebpf.LoadEBPFTracingProgram()
if err != nil {
return err
}
}
// We only allow up to 6 args for a BPF probe.
// We only allow up to 12 args for a BPF probe.
// 6 inputs + 6 outputs.
// Return early if we have more.
if len(args) > 6 {
return errors.New("too many arguments in traced function, max is 6")
if len(args) > 12 {
return errors.New("too many arguments in traced function, max is 12 input+return")
}
fn, ok := dbp.bi.LookupFunc[fnName]
@ -723,7 +729,7 @@ func (dbp *nativeProcess) SetUProbe(fnName string, goidOffset int64, args []ebpf
}
key := fn.Entry
err := dbp.os.ebpf.UpdateArgMap(key, goidOffset, args, dbp.BinInfo().GStructOffset())
err := dbp.os.ebpf.UpdateArgMap(key, goidOffset, args, dbp.BinInfo().GStructOffset(), false)
if err != nil {
return err
}
@ -733,6 +739,49 @@ func (dbp *nativeProcess) SetUProbe(fnName string, goidOffset int64, args []ebpf
if err != nil {
return err
}
// First attach a uprobe at all return addresses. We do this instead of using a uretprobe
// for two reasons:
// 1. uretprobes do not play well with Go
// 2. uretprobes seem to not restore the function return addr on the stack when removed, destroying any
// kind of workaround we could come up with.
// TODO(derekparker): this whole thing could likely be optimized a bit.
img := dbp.BinInfo().PCToImage(fn.Entry)
f, err := elf.Open(img.Path)
if err != nil {
return fmt.Errorf("could not open elf file to resolve symbol offset: %w", err)
}
var regs proc.Registers
mem := dbp.Memory()
regs, _ = dbp.memthread.Registers()
instructions, err := proc.Disassemble(mem, regs, &proc.BreakpointMap{}, dbp.BinInfo(), fn.Entry, fn.End)
if err != nil {
return err
}
var addrs []uint64
for _, instruction := range instructions {
if instruction.IsRet() {
addrs = append(addrs, instruction.Loc.PC)
}
}
addrs = append(addrs, proc.FindDeferReturnCalls(instructions)...)
for _, addr := range addrs {
err := dbp.os.ebpf.UpdateArgMap(addr, goidOffset, args, dbp.BinInfo().GStructOffset(), true)
if err != nil {
return err
}
off, err := ebpf.AddressToOffset(f, addr)
if err != nil {
return err
}
err = dbp.os.ebpf.AttachUprobe(dbp.Pid(), debugname, off)
if err != nil {
return err
}
}
return dbp.os.ebpf.AttachUprobe(dbp.Pid(), debugname, offset)
}

45
pkg/proc/target.go
View File

@ -10,6 +10,7 @@ import (
"github.com/go-delve/delve/pkg/dwarf/op"
"github.com/go-delve/delve/pkg/goversion"
"github.com/go-delve/delve/pkg/proc/internal/ebpf"
)
var (
@ -401,35 +402,45 @@ func (t *Target) CurrentThread() Thread {
}
type UProbeTraceResult struct {
FnAddr int
GoroutineID int
InputParams []*Variable
FnAddr int
GoroutineID int
InputParams []*Variable
ReturnParams []*Variable
}
func (t *Target) GetBufferedTracepoints() []*UProbeTraceResult {
var results []*UProbeTraceResult
tracepoints := t.proc.GetBufferedTracepoints()
convertInputParamToVariable := func(ip *ebpf.RawUProbeParam) *Variable {
v := &Variable{}
v.RealType = ip.RealType
v.Len = ip.Len
v.Base = ip.Base
v.Addr = ip.Addr
v.Kind = ip.Kind
cachedMem := CreateLoadedCachedMemory(ip.Data)
compMem, _ := CreateCompositeMemory(cachedMem, t.BinInfo().Arch, op.DwarfRegisters{}, ip.Pieces)
v.mem = compMem
// Load the value here so that we don't have to export
// loadValue outside of proc.
v.loadValue(loadFullValue)
return v
}
for _, tp := range tracepoints {
r := &UProbeTraceResult{}
r.FnAddr = tp.FnAddr
r.GoroutineID = tp.GoroutineID
for _, ip := range tp.InputParams {
v := &Variable{}
v.RealType = ip.RealType
v.Len = ip.Len
v.Base = ip.Base
v.Addr = ip.Addr
v.Kind = ip.Kind
cachedMem := CreateLoadedCachedMemory(ip.Data)
compMem, _ := CreateCompositeMemory(cachedMem, t.BinInfo().Arch, op.DwarfRegisters{}, ip.Pieces)
v.mem = compMem
// Load the value here so that we don't have to export
// loadValue outside of proc.
v.loadValue(loadFullValue)
v := convertInputParamToVariable(ip)
r.InputParams = append(r.InputParams, v)
}
for _, ip := range tp.ReturnParams {
v := convertInputParamToVariable(ip)
r.ReturnParams = append(r.ReturnParams, v)
}
results = append(results, r)
}
return results

3
service/debugger/debugger.go
View File

@ -2157,6 +2157,9 @@ func (d *Debugger) GetBufferedTracepoints() []api.TracepointResult {
for _, p := range trace.InputParams {
results[i].InputParams = append(results[i].InputParams, *api.ConvertVar(p))
}
for _, p := range trace.ReturnParams {
results[i].ReturnParams = append(results[i].ReturnParams, *api.ConvertVar(p))
}
}
return results
}