proc/tests: keep track of tests skipped due to backend problems (#2178)

* proc/tests: keep track of tests skipped due to backend problems

Mark tests skipped due to backend problems and add a script to keep
track of them.

* Travis-CI: add ignorechecksum option to chocolatey command

Looks like a configuration problem on chocolatey's end.

Documentation/backend_test_health.md

@@ -0,0 +1,19 @@
+Tests skipped by each supported backend:
+
+* 386 skipped = 1.4% (2/145)
+	* 2 broken - cgo stacktraces
+* arm64 skipped = 2.1% (3/145)
+	* 1 broken
+	* 1 broken - global variable symbolication
+	* 1 broken - reading defers
+* darwin/lldb skipped = 0.69% (1/145)
+	* 1 upstream issue
+* freebsd skipped = 7.6% (11/145)
+	* 11 broken
+* linux/386/pie skipped = 0.69% (1/145)
+	* 1 broken
+* pie skipped = 0.69% (1/145)
+	* 1 upstream issue - https://github.com/golang/go/issues/29322
+* windows skipped = 1.4% (2/145)
+	* 1 broken
+	* 1 upstream issue

Documentation/faq.md

@@ -3,16 +3,20 @@
 #### I'm getting an error while compiling Delve / unsupported architectures and OSs
 
 The most likely cause of this is that you are running an unsupported Operating System or architecture.
-Currently Delve supports Linux, Windows and macOS on the amd64 (also known as Intel 86x64) architecture and Linux on the arm64 (also known as ARM AArch64) and i386 architectures.
+Currently Delve supports (GOOS / GOARCH):
-
+* linux / amd64 (86x64)
-For example if you are getting the `undefined: ArchInst` error message while compiling Delve, that means that Delve doesn't currently support your processor architecture.
+* linux / arm64 (AARCH64)
+* linux / 386
+* windows / amd64
+* darwin (macOS) / amd64
 
 There is no planned ETA for support of other architectures or operating systems. Bugs tracking requested support are:
 
-- [i386 support](https://github.com/go-delve/delve/issues/20)
+- [32bit ARM support](https://github.com/go-delve/delve/issues/328)
-- [ARM support](https://github.com/go-delve/delve/issues/118)
 - [PowerPC support](https://github.com/go-delve/delve/issues/1564)
-- [FreeBSD support](https://github.com/go-delve/delve/issues/213)
+- [OpenBSD](https://github.com/go-delve/delve/issues/1477)
+
+See also: [backend test health](backend_test_health.md).
 
 #### How do I use Delve with Docker?
 

_scripts/gen-backend_test_health.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io"
+	"log"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+func main() {
+	f, err := parser.ParseFile(new(token.FileSet), "pkg/proc/proc_test.go", nil, 0)
+	if err != nil {
+		log.Fatalf("could not compile proc_test.go: %v", err)
+	}
+	ntests := 0
+	skipped := make(map[string]map[string]int)
+	ast.Inspect(f, func(node ast.Node) bool {
+		switch node := node.(type) {
+		case *ast.File:
+			return true
+		case *ast.FuncDecl:
+			if !strings.HasPrefix(node.Name.Name, "Test") {
+				return false
+			}
+			ntests++
+		stmtLoop:
+			for _, stmt := range node.Body.List {
+				expr, isexpr := stmt.(*ast.ExprStmt)
+				if !isexpr {
+					continue
+				}
+				call, ok := expr.X.(*ast.CallExpr)
+				if !ok {
+					return false
+				}
+				fun, ok := call.Fun.(*ast.Ident)
+				if !ok {
+					return false
+				}
+				switch fun.Name {
+				case "skipOn":
+					reason, conditions := skipOnArgs(call.Args)
+					if reason == "N/A" {
+						ntests--
+						break stmtLoop
+					}
+					if skipped[conditions] == nil {
+						skipped[conditions] = make(map[string]int)
+					}
+					skipped[conditions][reason]++
+				case "skipUnlessOn":
+					ntests--
+					break stmtLoop
+				}
+			}
+		}
+		return false
+	})
+	var fh io.WriteCloser
+	if len(os.Args) > 1 && os.Args[1] == "-" {
+		fh = os.Stdout
+	} else {
+		fh, err = os.Create("./Documentation/backend_test_health.md")
+		if err != nil {
+			log.Fatalf("could not create backend_test_health.md: %v", err)
+		}
+	}
+	fmt.Fprintf(fh, "Tests skipped by each supported backend:\n\n")
+	conds := []string{}
+	for cond := range skipped {
+		conds = append(conds, cond)
+	}
+	sort.Strings(conds)
+	for _, cond := range conds {
+		tot := 0
+		for _, v := range skipped[cond] {
+			tot += v
+		}
+		perc := float64(tot) / float64(ntests) * 100
+		fmt.Fprintf(fh, "* %s skipped = %0.02g%% (%d/%d)\n", cond, perc, tot, ntests)
+		reasons := []string{}
+		for reason := range skipped[cond] {
+			reasons = append(reasons, reason)
+		}
+		sort.Strings(reasons)
+		for _, reason := range reasons {
+			fmt.Fprintf(fh, "\t* %d %s\n", skipped[cond][reason], reason)
+		}
+	}
+	err = fh.Close()
+	if err != nil {
+		log.Fatalf("could not close output file: %v", err)
+	}
+}
+
+func skipOnArgs(args []ast.Expr) (reason string, conditions string) {
+	reason, _ = strconv.Unquote(args[1].(*ast.BasicLit).Value)
+	conds := []string{}
+	for _, arg := range args[2:] {
+		cond, _ := strconv.Unquote(arg.(*ast.BasicLit).Value)
+		conds = append(conds, cond)
+	}
+	conditions = strings.Join(conds, "/")
+	return reason, conditions
+}

cmd/dlv/dlv_test.go

@@ -359,6 +359,7 @@ func TestGeneratedDoc(t *testing.T) {
 
 	checkAutogenDoc(t, "pkg/terminal/starbind/starlark_mapping.go", "'go generate' inside pkg/terminal/starbind", runScript("_scripts/gen-starlark-bindings.go", "go", "-"))
 	checkAutogenDoc(t, "Documentation/cli/starlark.md", "'go generate' inside pkg/terminal/starbind", runScript("_scripts/gen-starlark-bindings.go", "doc/dummy", "Documentation/cli/starlark.md"))
+	checkAutogenDoc(t, "Documentation/backend_test_health.md", "go run _scripts/gen-backend_test_health.go", runScript("_scripts/gen-backend_test_health.go", "-"))
 }
 
 func TestExitInInit(t *testing.T) {

pkg/proc/proc_test.go

@@ -54,6 +54,34 @@ func TestMain(m *testing.M) {
 	os.Exit(protest.RunTestsWithFixtures(m))
 }
 
+func matchSkipConditions(conditions ...string) bool {
+	for _, cond := range conditions {
+		condfound := false
+		for _, s := range []string{runtime.GOOS, runtime.GOARCH, testBackend, buildMode} {
+			if s == cond {
+				condfound = true
+				break
+			}
+		}
+		if !condfound {
+			return false
+		}
+	}
+	return true
+}
+
+func skipOn(t testing.TB, reason string, conditions ...string) {
+	if matchSkipConditions(conditions...) {
+		t.Skipf("skipped on %s: %s", strings.Join(conditions, "/"), reason)
+	}
+}
+
+func skipUnlessOn(t testing.TB, reason string, conditions ...string) {
+	if !matchSkipConditions(conditions...) {
+		t.Skipf("skipped on %s: %s", strings.Join(conditions, "/"), reason)
+	}
+}
+
 func withTestProcess(name string, t testing.TB, fn func(p *proc.Target, fixture protest.Fixture)) {
 	withTestProcessArgs(name, t, ".", []string{}, 0, fn)
 }
@@ -551,9 +579,7 @@ func TestNextGeneral(t *testing.T) {
 }
 
 func TestNextConcurrent(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
 	testcases := []nextTest{
 		{8, 9},
 		{9, 10},
@@ -589,9 +615,7 @@ func TestNextConcurrent(t *testing.T) {
 }
 
 func TestNextConcurrentVariant2(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
 	// Just like TestNextConcurrent but instead of removing the initial breakpoint we check that when it happens is for other goroutines
 	testcases := []nextTest{
 		{8, 9},
@@ -804,8 +828,8 @@ func TestSwitchThread(t *testing.T) {
 func TestCGONext(t *testing.T) {
 	// Test if one can do 'next' in a cgo binary
 	// On OSX with Go < 1.5 CGO is not supported due to: https://github.com/golang/go/issues/8973
-	if runtime.GOOS == "darwin" && strings.Contains(runtime.Version(), "1.4") {
+	if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 5) {
-		return
+		skipOn(t, "upstream issue", "darwin")
 	}
 	protest.MustHaveCgo(t)
 
@@ -997,10 +1021,7 @@ func TestStacktraceGoroutine(t *testing.T) {
 }
 
 func TestKill(t *testing.T) {
-	if testBackend == "lldb" {
+	skipOn(t, "N/A", "lldb") // k command presumably works but leaves the process around?
-		// k command presumably works but leaves the process around?
-		return
-	}
 	withTestProcess("testprog", t, func(p *proc.Target, fixture protest.Fixture) {
 		if err := p.Detach(true); err != nil {
 			t.Fatal(err)
@@ -1044,8 +1065,8 @@ func TestGetG(t *testing.T) {
 	})
 
 	// On OSX with Go < 1.5 CGO is not supported due to: https://github.com/golang/go/issues/8973
-	if runtime.GOOS == "darwin" && strings.Contains(runtime.Version(), "1.4") {
+	if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 5) {
-		return
+		skipOn(t, "upstream issue", "darwin")
 	}
 	protest.MustHaveCgo(t)
 
@@ -1414,9 +1435,7 @@ func TestIssue325(t *testing.T) {
 }
 
 func TestBreakpointCounts(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
 	protest.AllowRecording(t)
 	withTestProcess("bpcountstest", t, func(p *proc.Target, fixture protest.Fixture) {
 		bp := setFileBreakpoint(p, t, fixture.Source, 12)
@@ -1622,9 +1641,7 @@ func BenchmarkLocalVariables(b *testing.B) {
 }
 
 func TestCondBreakpoint(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
 	protest.AllowRecording(t)
 	withTestProcess("parallel_next", t, func(p *proc.Target, fixture protest.Fixture) {
 		bp := setFileBreakpoint(p, t, fixture.Source, 9)
@@ -1646,9 +1663,7 @@ func TestCondBreakpoint(t *testing.T) {
 }
 
 func TestCondBreakpointError(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
 	protest.AllowRecording(t)
 	withTestProcess("parallel_next", t, func(p *proc.Target, fixture protest.Fixture) {
 		bp := setFileBreakpoint(p, t, fixture.Source, 9)
@@ -1819,9 +1834,7 @@ func TestIssue396(t *testing.T) {
 }
 
 func TestIssue414(t *testing.T) {
-	if runtime.GOOS == "linux" && runtime.GOARCH == "386" && buildMode == "pie" {
+	skipOn(t, "broken", "linux", "386", "pie") // test occasionally hangs on linux/386/pie
-		t.Skip("test occasionally hangs on linux/386/pie")
-	}
 	// Stepping until the program exits
 	protest.AllowRecording(t)
 	withTestProcess("math", t, func(p *proc.Target, fixture protest.Fixture) {
@@ -1946,10 +1959,7 @@ func TestCmdLineArgs(t *testing.T) {
 }
 
 func TestIssue462(t *testing.T) {
-	// Stacktrace of Goroutine 0 fails with an error
+	skipOn(t, "broken", "windows") // Stacktrace of Goroutine 0 fails with an error
-	if runtime.GOOS == "windows" {
-		return
-	}
 	withTestProcess("testnextnethttp", t, func(p *proc.Target, fixture protest.Fixture) {
 		go func() {
 			// Wait for program to start listening.
@@ -1972,9 +1982,7 @@ func TestIssue462(t *testing.T) {
 }
 
 func TestNextParked(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
 	protest.AllowRecording(t)
 	withTestProcess("parallel_next", t, func(p *proc.Target, fixture protest.Fixture) {
 		bp := setFunctionBreakpoint(p, t, "main.sayhi")
@@ -2025,9 +2033,7 @@ func TestNextParked(t *testing.T) {
 }
 
 func TestStepParked(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
 	protest.AllowRecording(t)
 	withTestProcess("parallel_next", t, func(p *proc.Target, fixture protest.Fixture) {
 		bp := setFunctionBreakpoint(p, t, "main.sayhi")
@@ -2343,9 +2349,7 @@ func TestStepOut(t *testing.T) {
 }
 
 func TestStepConcurrentDirect(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
 	protest.AllowRecording(t)
 	withTestProcess("teststepconcurrent", t, func(p *proc.Target, fixture protest.Fixture) {
 		bp := setFileBreakpoint(p, t, fixture.Source, 37)
@@ -2409,9 +2413,7 @@ func TestStepConcurrentDirect(t *testing.T) {
 }
 
 func TestStepConcurrentPtr(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
 	protest.AllowRecording(t)
 	withTestProcess("teststepconcurrent", t, func(p *proc.Target, fixture protest.Fixture) {
 		setFileBreakpoint(p, t, fixture.Source, 24)
@@ -2553,14 +2555,13 @@ func TestStepOnCallPtrInstr(t *testing.T) {
 }
 
 func TestIssue594(t *testing.T) {
-	if runtime.GOOS == "darwin" && testBackend == "lldb" {
+	skipOn(t, "upstream issue", "darwin", "lldb")
 	// debugserver will receive an EXC_BAD_ACCESS for this, at that point
 	// there is no way to reconvert this exception into a unix signal and send
 	// it to the process.
 	// This is a bug in debugserver/lldb:
 	//  https://bugs.llvm.org//show_bug.cgi?id=22868
-		return
+
-	}
 	// Exceptions that aren't caused by breakpoints should be propagated
 	// back to the target.
 	// In particular the target should be able to cause a nil pointer
@@ -3477,10 +3478,7 @@ func TestSystemstackOnRuntimeNewstack(t *testing.T) {
 }
 
 func TestIssue1034(t *testing.T) {
-	if runtime.GOARCH == "386" {
+	skipOn(t, "broken - cgo stacktraces", "386")
-		t.Skip("cgo stacktraces not supported on i386 for now")
-	}
-
 	protest.MustHaveCgo(t)
 
 	// The external linker on macOS produces an abbrev for DW_TAG_subprogram
@@ -3500,10 +3498,7 @@ func TestIssue1034(t *testing.T) {
 }
 
 func TestIssue1008(t *testing.T) {
-	if runtime.GOARCH == "386" {
+	skipOn(t, "broken - cgo stacktraces", "386")
-		t.Skip("cgo stacktraces not supported on i386 for now")
-	}
-
 	protest.MustHaveCgo(t)
 
 	// The external linker on macOS inserts "end of sequence" extended opcodes
@@ -3656,9 +3651,7 @@ func TestIssue1145(t *testing.T) {
 }
 
 func TestDisassembleGlobalVars(t *testing.T) {
-	if runtime.GOARCH == "arm64" {
+	skipOn(t, "broken - global variable symbolication", "arm64") // On ARM64 symLookup can't look up variables due to how they are loaded, see issue #1778
-		t.Skip("On ARM64 symLookup can't look up variables due to how they are loaded, see issue #1778")
-	}
 	// On 386 linux when pie, the genered code use __x86.get_pc_thunk to ensure position-independent.
 	// Locate global variable by
 	//    `CALL __x86.get_pc_thunk.ax(SB) 0xb0f7f
@@ -3973,9 +3966,7 @@ func TestIssue951(t *testing.T) {
 }
 
 func TestDWZCompression(t *testing.T) {
-	if runtime.GOARCH == "arm64" {
+	skipOn(t, "broken", "arm64")
-		t.Skip("test is not valid on ARM64")
-	}
 	// If dwz is not available in the system, skip this test
 	if _, err := exec.LookPath("dwz"); err != nil {
 		t.Skip("dwz not installed")
@@ -4157,9 +4148,7 @@ func TestReadDefer(t *testing.T) {
 }
 
 func TestNextUnknownInstr(t *testing.T) {
-	if runtime.GOARCH != "amd64" {
+	skipUnlessOn(t, "amd64 only", "amd64")
-		t.Skip("amd64 only")
-	}
 	if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 10) {
 		t.Skip("versions of Go before 1.10 can't assemble the instruction VPUNPCKLWD")
 	}
@@ -4171,9 +4160,7 @@ func TestNextUnknownInstr(t *testing.T) {
 }
 
 func TestReadDeferArgs(t *testing.T) {
-	if runtime.GOARCH == "arm64" {
+	skipOn(t, "broken - reading defers", "arm64")
-		t.Skip("arm64 does not support ReadDeferArgs for now")
-	}
 	var tests = []struct {
 		frame, deferCall int
 		a, b             int64
@@ -4313,9 +4300,7 @@ func TestIssue1469(t *testing.T) {
 }
 
 func TestDeadlockBreakpoint(t *testing.T) {
-	if buildMode == "pie" {
+	skipOn(t, "upstream issue - https://github.com/golang/go/issues/29322", "pie")
-		t.Skip("See https://github.com/golang/go/issues/29322")
-	}
 	deadlockBp := proc.FatalThrow
 	if !goversion.VersionAfterOrEqual(runtime.Version(), 1, 11) {
 		deadlockBp = proc.UnrecoveredPanic
@@ -4450,10 +4435,7 @@ func testCallConcurrentCheckReturns(p *proc.Target, t *testing.T, gid1, gid2 int
 }
 
 func TestCallConcurrent(t *testing.T) {
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("test is not valid on FreeBSD")
-	}
-
 	protest.MustSupportFunctionCalls(t, testBackend)
 	withTestProcess("teststepconcurrent", t, func(p *proc.Target, fixture protest.Fixture) {
 		bp := setFileBreakpoint(p, t, fixture.Source, 24)
@@ -4536,9 +4518,7 @@ func TestIssue1615(t *testing.T) {
 }
 
 func TestCgoStacktrace2(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	skipOn(t, "upstream issue", "windows")
-		t.Skip("fixture crashes go runtime on windows")
-	}
 	protest.MustHaveCgo(t)
 	// If a panic happens during cgo execution the stacktrace should show the C
 	// function that caused the problem.
@@ -4552,9 +4532,7 @@ func TestCgoStacktrace2(t *testing.T) {
 }
 
 func TestIssue1656(t *testing.T) {
-	if runtime.GOARCH != "amd64" {
+	skipUnlessOn(t, "amd64 only", "amd64")
-		t.Skip("amd64 only")
-	}
 	withTestProcess("issue1656/", t, func(p *proc.Target, fixture protest.Fixture) {
 		setFileBreakpoint(p, t, filepath.ToSlash(filepath.Join(fixture.BuildDir, "main.s")), 5)
 		assertNoError(p.Continue(), t, "Continue()")
@@ -4572,9 +4550,7 @@ func TestBreakpointConfusionOnResume(t *testing.T) {
 	// native.(*Thread).singleStep all agree on which breakpoint the thread is
 	// stopped at.
 	// This test checks for a regression introduced when fixing Issue #1656
-	if runtime.GOARCH != "amd64" {
+	skipUnlessOn(t, "amd64 only", "amd64")
-		t.Skip("amd64 only")
-	}
 	withTestProcess("nopbreakpoint/", t, func(p *proc.Target, fixture protest.Fixture) {
 		maindots := filepath.ToSlash(filepath.Join(fixture.BuildDir, "main.s"))
 		maindotgo := filepath.ToSlash(filepath.Join(fixture.BuildDir, "main.go"))
@@ -4823,9 +4799,7 @@ func TestIssue1925(t *testing.T) {
 }
 
 func TestStepIntoWrapperForEmbeddedPointer(t *testing.T) {
-	if runtime.GOOS == "linux" && runtime.GOARCH == "386" && buildMode == "pie" {
+	skipOn(t, "N/A", "linux", "386", "pie") // skipping wrappers doesn't work on linux/386/PIE due to the use of get_pc_thunk
-		t.Skip("Skipping wrappers doesn't work on linux/386/PIE due to the use of get_pc_thunk")
-	}
 	// Under some circumstances (when using an interface to call a method on an
 	// embedded field, see _fixtures/ifaceembcall.go) the compiler will
 	// autogenerate a wrapper function that uses a tail call (i.e. it ends in
@@ -4874,9 +4848,7 @@ func TestRefreshCurThreadSelGAfterContinueOnceError(t *testing.T) {
 	// refreshed after ContinueOnce returns an error due to a segmentation
 	// fault.
 
-	if runtime.GOOS != "darwin" && testBackend != "lldb" {
+	skipUnlessOn(t, "N/A", "darwin", "lldb")
-		t.Skip("not applicable")
-	}
 
 	withTestProcess("issue2078", t, func(p *proc.Target, fixture protest.Fixture) {
 		setFileBreakpoint(p, t, fixture.Source, 4)
@@ -4942,9 +4914,7 @@ func TestRequestManualStopWhileStopped(t *testing.T) {
 
 func TestStepOutPreservesGoroutine(t *testing.T) {
 	// Checks that StepOut preserves the currently selected goroutine.
-	if runtime.GOOS == "freebsd" {
+	skipOn(t, "broken", "freebsd")
-		t.Skip("XXX - not working")
-	}
 	rand.Seed(time.Now().Unix())
 	withTestProcess("issue2113", t, func(p *proc.Target, fixture protest.Fixture) {
 		assertNoError(p.Continue(), t, "Continue()")