PenaDevops/delve
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

proc/tests: keep track of tests skipped due to backend problems (#2178)

Browse Source 
* proc/tests: keep track of tests skipped due to backend problems

Mark tests skipped due to backend problems and add a script to keep
track of them.

* Travis-CI: add ignorechecksum option to chocolatey command

Looks like a configuration problem on chocolatey's end.
This commit is contained in:
Alessandro Arzilli 2020-09-24 17:10:20 +02:00 committed by GitHub
parent 7e00666b9f
commit 84328ed870
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 206 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
Documentation/backend_test_health.md Normal file
View File

@ -0,0 +1,19 @@
Tests skipped by each supported backend:
* 386 skipped = 1.4% (2/145)
* 2 broken - cgo stacktraces
* arm64 skipped = 2.1% (3/145)
* 1 broken
* 1 broken - global variable symbolication
* 1 broken - reading defers
* darwin/lldb skipped = 0.69% (1/145)
* 1 upstream issue
* freebsd skipped = 7.6% (11/145)
* 11 broken
* linux/386/pie skipped = 0.69% (1/145)
* 1 broken
* pie skipped = 0.69% (1/145)
* 1 upstream issue - https://github.com/golang/go/issues/29322
* windows skipped = 1.4% (2/145)
* 1 broken
* 1 upstream issue

16
Documentation/faq.md
View File

@ -3,16 +3,20 @@
#### I'm getting an error while compiling Delve / unsupported architectures and OSs
The most likely cause of this is that you are running an unsupported Operating System or architecture.
Currently Delve supports Linux, Windows and macOS on the amd64 (also known as Intel 86x64) architecture and Linux on the arm64 (also known as ARM AArch64) and i386 architectures.
For example if you are getting the `undefined: ArchInst` error message while compiling Delve, that means that Delve doesn't currently support your processor architecture.
Currently Delve supports (GOOS / GOARCH):
* linux / amd64 (86x64)
* linux / arm64 (AARCH64)
* linux / 386
* windows / amd64
* darwin (macOS) / amd64
There is no planned ETA for support of other architectures or operating systems. Bugs tracking requested support are:
- [i386 support](https://github.com/go-delve/delve/issues/20)
- [ARM support](https://github.com/go-delve/delve/issues/118)
- [32bit ARM support](https://github.com/go-delve/delve/issues/328)
- [PowerPC support](https://github.com/go-delve/delve/issues/1564)
- [FreeBSD support](https://github.com/go-delve/delve/issues/213)
- [OpenBSD](https://github.com/go-delve/delve/issues/1477)
See also: [backend test health](backend_test_health.md).
#### How do I use Delve with Docker?

BIN
_scripts/backend_test_health Executable file
View File

Binary file not shown.

111
_scripts/gen-backend_test_health.go Normal file
View File

@ -0,0 +1,111 @@
package main
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"io"
"log"
"os"
"sort"
"strconv"
"strings"
)
func main() {
f, err := parser.ParseFile(new(token.FileSet), "pkg/proc/proc_test.go", nil, 0)
if err != nil {
log.Fatalf("could not compile proc_test.go: %v", err)
}
ntests := 0
skipped := make(map[string]map[string]int)
ast.Inspect(f, func(node ast.Node) bool {
switch node := node.(type) {
case *ast.File:
return true
case *ast.FuncDecl:
if !strings.HasPrefix(node.Name.Name, "Test") {
return false
}
ntests++
stmtLoop:
for _, stmt := range node.Body.List {
expr, isexpr := stmt.(*ast.ExprStmt)
if !isexpr {
continue
}
call, ok := expr.X.(*ast.CallExpr)
if !ok {
return false
}
fun, ok := call.Fun.(*ast.Ident)
if !ok {
return false
}
switch fun.Name {
case "skipOn":
reason, conditions := skipOnArgs(call.Args)
if reason == "N/A" {
ntests--
break stmtLoop
}
if skipped[conditions] == nil {
skipped[conditions] = make(map[string]int)
}
skipped[conditions][reason]++
case "skipUnlessOn":
ntests--
break stmtLoop
}
}
}
return false
})
var fh io.WriteCloser
if len(os.Args) > 1 && os.Args[1] == "-" {
fh = os.Stdout
} else {
fh, err = os.Create("./Documentation/backend_test_health.md")
if err != nil {
log.Fatalf("could not create backend_test_health.md: %v", err)
}
}
fmt.Fprintf(fh, "Tests skipped by each supported backend:\n\n")
conds := []string{}
for cond := range skipped {
conds = append(conds, cond)
}
sort.Strings(conds)
for _, cond := range conds {
tot := 0
for _, v := range skipped[cond] {
tot += v
}
perc := float64(tot) / float64(ntests) * 100
fmt.Fprintf(fh, "* %s skipped = %0.02g%% (%d/%d)\n", cond, perc, tot, ntests)
reasons := []string{}
for reason := range skipped[cond] {
reasons = append(reasons, reason)
}
sort.Strings(reasons)
for _, reason := range reasons {
fmt.Fprintf(fh, "\t* %d %s\n", skipped[cond][reason], reason)
}
}
err = fh.Close()
if err != nil {
log.Fatalf("could not close output file: %v", err)
}
}
func skipOnArgs(args []ast.Expr) (reason string, conditions string) {
reason, _ = strconv.Unquote(args[1].(*ast.BasicLit).Value)
conds := []string{}
for _, arg := range args[2:] {
cond, _ := strconv.Unquote(arg.(*ast.BasicLit).Value)
conds = append(conds, cond)
}
conditions = strings.Join(conds, "/")
return reason, conditions
}

1
cmd/dlv/dlv_test.go
View File

@ -359,6 +359,7 @@ func TestGeneratedDoc(t *testing.T) {
checkAutogenDoc(t, "pkg/terminal/starbind/starlark_mapping.go", "'go generate' inside pkg/terminal/starbind", runScript("_scripts/gen-starlark-bindings.go", "go", "-"))
checkAutogenDoc(t, "Documentation/cli/starlark.md", "'go generate' inside pkg/terminal/starbind", runScript("_scripts/gen-starlark-bindings.go", "doc/dummy", "Documentation/cli/starlark.md"))
checkAutogenDoc(t, "Documentation/backend_test_health.md", "go run _scripts/gen-backend_test_health.go", runScript("_scripts/gen-backend_test_health.go", "-"))
}
func TestExitInInit(t *testing.T) {

160
pkg/proc/proc_test.go
View File

@ -54,6 +54,34 @@ func TestMain(m *testing.M) {
os.Exit(protest.RunTestsWithFixtures(m))
}
func matchSkipConditions(conditions ...string) bool {
for _, cond := range conditions {
condfound := false
for _, s := range []string{runtime.GOOS, runtime.GOARCH, testBackend, buildMode} {
if s == cond {
condfound = true
break
}
}
if !condfound {
return false
}
}
return true
}
func skipOn(t testing.TB, reason string, conditions ...string) {
if matchSkipConditions(conditions...) {
t.Skipf("skipped on %s: %s", strings.Join(conditions, "/"), reason)
}
}
func skipUnlessOn(t testing.TB, reason string, conditions ...string) {
if !matchSkipConditions(conditions...) {
t.Skipf("skipped on %s: %s", strings.Join(conditions, "/"), reason)
}
}
func withTestProcess(name string, t testing.TB, fn func(p *proc.Target, fixture protest.Fixture)) {
withTestProcessArgs(name, t, ".", []string{}, 0, fn)
}
@ -551,9 +579,7 @@ func TestNextGeneral(t *testing.T) {
}
func TestNextConcurrent(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
testcases := []nextTest{
{8, 9},
{9, 10},
@ -589,9 +615,7 @@ func TestNextConcurrent(t *testing.T) {
}
func TestNextConcurrentVariant2(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
// Just like TestNextConcurrent but instead of removing the initial breakpoint we check that when it happens is for other goroutines
testcases := []nextTest{
{8, 9},
@ -804,8 +828,8 @@ func TestSwitchThread(t *testing.T) {
func TestCGONext(t *testing.T) {
// Test if one can do 'next' in a cgo binary
// On OSX with Go < 1.5 CGO is not supported due to: https://github.com/golang/go/issues/8973
if runtime.GOOS == "darwin" && strings.Contains(runtime.Version(), "1.4") {
return
if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 5) {
skipOn(t, "upstream issue", "darwin")
}
protest.MustHaveCgo(t)
@ -997,10 +1021,7 @@ func TestStacktraceGoroutine(t *testing.T) {
}
func TestKill(t *testing.T) {
if testBackend == "lldb" {
// k command presumably works but leaves the process around?
return
}
skipOn(t, "N/A", "lldb") // k command presumably works but leaves the process around?
withTestProcess("testprog", t, func(p *proc.Target, fixture protest.Fixture) {
if err := p.Detach(true); err != nil {
t.Fatal(err)
@ -1044,8 +1065,8 @@ func TestGetG(t *testing.T) {
})
// On OSX with Go < 1.5 CGO is not supported due to: https://github.com/golang/go/issues/8973
if runtime.GOOS == "darwin" && strings.Contains(runtime.Version(), "1.4") {
return
if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 5) {
skipOn(t, "upstream issue", "darwin")
}
protest.MustHaveCgo(t)
@ -1414,9 +1435,7 @@ func TestIssue325(t *testing.T) {
}
func TestBreakpointCounts(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
protest.AllowRecording(t)
withTestProcess("bpcountstest", t, func(p *proc.Target, fixture protest.Fixture) {
bp := setFileBreakpoint(p, t, fixture.Source, 12)
@ -1622,9 +1641,7 @@ func BenchmarkLocalVariables(b *testing.B) {
}
func TestCondBreakpoint(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
protest.AllowRecording(t)
withTestProcess("parallel_next", t, func(p *proc.Target, fixture protest.Fixture) {
bp := setFileBreakpoint(p, t, fixture.Source, 9)
@ -1646,9 +1663,7 @@ func TestCondBreakpoint(t *testing.T) {
}
func TestCondBreakpointError(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
protest.AllowRecording(t)
withTestProcess("parallel_next", t, func(p *proc.Target, fixture protest.Fixture) {
bp := setFileBreakpoint(p, t, fixture.Source, 9)
@ -1819,9 +1834,7 @@ func TestIssue396(t *testing.T) {
}
func TestIssue414(t *testing.T) {
if runtime.GOOS == "linux" && runtime.GOARCH == "386" && buildMode == "pie" {
t.Skip("test occasionally hangs on linux/386/pie")
}
skipOn(t, "broken", "linux", "386", "pie") // test occasionally hangs on linux/386/pie
// Stepping until the program exits
protest.AllowRecording(t)
withTestProcess("math", t, func(p *proc.Target, fixture protest.Fixture) {
@ -1946,10 +1959,7 @@ func TestCmdLineArgs(t *testing.T) {
}
func TestIssue462(t *testing.T) {
// Stacktrace of Goroutine 0 fails with an error
if runtime.GOOS == "windows" {
return
}
skipOn(t, "broken", "windows") // Stacktrace of Goroutine 0 fails with an error
withTestProcess("testnextnethttp", t, func(p *proc.Target, fixture protest.Fixture) {
go func() {
// Wait for program to start listening.
@ -1972,9 +1982,7 @@ func TestIssue462(t *testing.T) {
}
func TestNextParked(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
protest.AllowRecording(t)
withTestProcess("parallel_next", t, func(p *proc.Target, fixture protest.Fixture) {
bp := setFunctionBreakpoint(p, t, "main.sayhi")
@ -2025,9 +2033,7 @@ func TestNextParked(t *testing.T) {
}
func TestStepParked(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
protest.AllowRecording(t)
withTestProcess("parallel_next", t, func(p *proc.Target, fixture protest.Fixture) {
bp := setFunctionBreakpoint(p, t, "main.sayhi")
@ -2343,9 +2349,7 @@ func TestStepOut(t *testing.T) {
}
func TestStepConcurrentDirect(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
protest.AllowRecording(t)
withTestProcess("teststepconcurrent", t, func(p *proc.Target, fixture protest.Fixture) {
bp := setFileBreakpoint(p, t, fixture.Source, 37)
@ -2409,9 +2413,7 @@ func TestStepConcurrentDirect(t *testing.T) {
}
func TestStepConcurrentPtr(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
protest.AllowRecording(t)
withTestProcess("teststepconcurrent", t, func(p *proc.Target, fixture protest.Fixture) {
setFileBreakpoint(p, t, fixture.Source, 24)
@ -2553,14 +2555,13 @@ func TestStepOnCallPtrInstr(t *testing.T) {
}
func TestIssue594(t *testing.T) {
if runtime.GOOS == "darwin" && testBackend == "lldb" {
// debugserver will receive an EXC_BAD_ACCESS for this, at that point
// there is no way to reconvert this exception into a unix signal and send
// it to the process.
// This is a bug in debugserver/lldb:
// https://bugs.llvm.org//show_bug.cgi?id=22868
return
}
skipOn(t, "upstream issue", "darwin", "lldb")
// debugserver will receive an EXC_BAD_ACCESS for this, at that point
// there is no way to reconvert this exception into a unix signal and send
// it to the process.
// This is a bug in debugserver/lldb:
// https://bugs.llvm.org//show_bug.cgi?id=22868
// Exceptions that aren't caused by breakpoints should be propagated
// back to the target.
// In particular the target should be able to cause a nil pointer
@ -3477,10 +3478,7 @@ func TestSystemstackOnRuntimeNewstack(t *testing.T) {
}
func TestIssue1034(t *testing.T) {
if runtime.GOARCH == "386" {
t.Skip("cgo stacktraces not supported on i386 for now")
}
skipOn(t, "broken - cgo stacktraces", "386")
protest.MustHaveCgo(t)
// The external linker on macOS produces an abbrev for DW_TAG_subprogram
@ -3500,10 +3498,7 @@ func TestIssue1034(t *testing.T) {
}
func TestIssue1008(t *testing.T) {
if runtime.GOARCH == "386" {
t.Skip("cgo stacktraces not supported on i386 for now")
}
skipOn(t, "broken - cgo stacktraces", "386")
protest.MustHaveCgo(t)
// The external linker on macOS inserts "end of sequence" extended opcodes
@ -3656,9 +3651,7 @@ func TestIssue1145(t *testing.T) {
}
func TestDisassembleGlobalVars(t *testing.T) {
if runtime.GOARCH == "arm64" {
t.Skip("On ARM64 symLookup can't look up variables due to how they are loaded, see issue #1778")
}
skipOn(t, "broken - global variable symbolication", "arm64") // On ARM64 symLookup can't look up variables due to how they are loaded, see issue #1778
// On 386 linux when pie, the genered code use __x86.get_pc_thunk to ensure position-independent.
// Locate global variable by
// `CALL __x86.get_pc_thunk.ax(SB) 0xb0f7f
@ -3973,9 +3966,7 @@ func TestIssue951(t *testing.T) {
}
func TestDWZCompression(t *testing.T) {
if runtime.GOARCH == "arm64" {
t.Skip("test is not valid on ARM64")
}
skipOn(t, "broken", "arm64")
// If dwz is not available in the system, skip this test
if _, err := exec.LookPath("dwz"); err != nil {
t.Skip("dwz not installed")
@ -4157,9 +4148,7 @@ func TestReadDefer(t *testing.T) {
}
func TestNextUnknownInstr(t *testing.T) {
if runtime.GOARCH != "amd64" {
t.Skip("amd64 only")
}
skipUnlessOn(t, "amd64 only", "amd64")
if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 10) {
t.Skip("versions of Go before 1.10 can't assemble the instruction VPUNPCKLWD")
}
@ -4171,9 +4160,7 @@ func TestNextUnknownInstr(t *testing.T) {
}
func TestReadDeferArgs(t *testing.T) {
if runtime.GOARCH == "arm64" {
t.Skip("arm64 does not support ReadDeferArgs for now")
}
skipOn(t, "broken - reading defers", "arm64")
var tests = []struct {
frame, deferCall int
a, b int64
@ -4313,9 +4300,7 @@ func TestIssue1469(t *testing.T) {
}
func TestDeadlockBreakpoint(t *testing.T) {
if buildMode == "pie" {
t.Skip("See https://github.com/golang/go/issues/29322")
}
skipOn(t, "upstream issue - https://github.com/golang/go/issues/29322", "pie")
deadlockBp := proc.FatalThrow
if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 11) {
deadlockBp = proc.UnrecoveredPanic
@ -4450,10 +4435,7 @@ func testCallConcurrentCheckReturns(p *proc.Target, t *testing.T, gid1, gid2 int
}
func TestCallConcurrent(t *testing.T) {
if runtime.GOOS == "freebsd" {
t.Skip("test is not valid on FreeBSD")
}
skipOn(t, "broken", "freebsd")
protest.MustSupportFunctionCalls(t, testBackend)
withTestProcess("teststepconcurrent", t, func(p *proc.Target, fixture protest.Fixture) {
bp := setFileBreakpoint(p, t, fixture.Source, 24)
@ -4536,9 +4518,7 @@ func TestIssue1615(t *testing.T) {
}
func TestCgoStacktrace2(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("fixture crashes go runtime on windows")
}
skipOn(t, "upstream issue", "windows")
protest.MustHaveCgo(t)
// If a panic happens during cgo execution the stacktrace should show the C
// function that caused the problem.
@ -4552,9 +4532,7 @@ func TestCgoStacktrace2(t *testing.T) {
}
func TestIssue1656(t *testing.T) {
if runtime.GOARCH != "amd64" {
t.Skip("amd64 only")
}
skipUnlessOn(t, "amd64 only", "amd64")
withTestProcess("issue1656/", t, func(p *proc.Target, fixture protest.Fixture) {
setFileBreakpoint(p, t, filepath.ToSlash(filepath.Join(fixture.BuildDir, "main.s")), 5)
assertNoError(p.Continue(), t, "Continue()")
@ -4572,9 +4550,7 @@ func TestBreakpointConfusionOnResume(t *testing.T) {
// native.(*Thread).singleStep all agree on which breakpoint the thread is
// stopped at.
// This test checks for a regression introduced when fixing Issue #1656
if runtime.GOARCH != "amd64" {
t.Skip("amd64 only")
}
skipUnlessOn(t, "amd64 only", "amd64")
withTestProcess("nopbreakpoint/", t, func(p *proc.Target, fixture protest.Fixture) {
maindots := filepath.ToSlash(filepath.Join(fixture.BuildDir, "main.s"))
maindotgo := filepath.ToSlash(filepath.Join(fixture.BuildDir, "main.go"))
@ -4823,9 +4799,7 @@ func TestIssue1925(t *testing.T) {
}
func TestStepIntoWrapperForEmbeddedPointer(t *testing.T) {
if runtime.GOOS == "linux" && runtime.GOARCH == "386" && buildMode == "pie" {
t.Skip("Skipping wrappers doesn't work on linux/386/PIE due to the use of get_pc_thunk")
}
skipOn(t, "N/A", "linux", "386", "pie") // skipping wrappers doesn't work on linux/386/PIE due to the use of get_pc_thunk
// Under some circumstances (when using an interface to call a method on an
// embedded field, see _fixtures/ifaceembcall.go) the compiler will
// autogenerate a wrapper function that uses a tail call (i.e. it ends in
@ -4874,9 +4848,7 @@ func TestRefreshCurThreadSelGAfterContinueOnceError(t *testing.T) {
// refreshed after ContinueOnce returns an error due to a segmentation
// fault.
if runtime.GOOS != "darwin" && testBackend != "lldb" {
t.Skip("not applicable")
}
skipUnlessOn(t, "N/A", "darwin", "lldb")
withTestProcess("issue2078", t, func(p *proc.Target, fixture protest.Fixture) {
setFileBreakpoint(p, t, fixture.Source, 4)
@ -4942,9 +4914,7 @@ func TestRequestManualStopWhileStopped(t *testing.T) {
func TestStepOutPreservesGoroutine(t *testing.T) {
// Checks that StepOut preserves the currently selected goroutine.
if runtime.GOOS == "freebsd" {
t.Skip("XXX - not working")
}
skipOn(t, "broken", "freebsd")
rand.Seed(time.Now().Unix())
withTestProcess("issue2113", t, func(p *proc.Target, fixture protest.Fixture) {
assertNoError(p.Continue(), t, "Continue()")