Prow testing

.prow.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+# A Prow job can override these defaults, but this shouldn't be necessary.
+
+# Only these tests make sense for csi-sanity.
+: ${CSI_PROW_TESTS:="unit"}
+
+. release-tools/prow.sh
+
+main

release-tools/.prow.sh

@@ -0,0 +1,7 @@
+#! /bin/bash -e
+#
+# This is for testing csi-release-tools itself in Prow. All other
+# repos use prow.sh for that, but as csi-release-tools isn't a normal
+# repo with some Go code in it, it has a custom Prow test script.
+
+./verify-shellcheck.sh "$(pwd)"

release-tools/README.md

@@ -49,3 +49,60 @@ Cheat sheet:
 - `git subtree add --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once)
 - `git subtree pull --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes)
 - edit, `git commit`, `git subtree push --prefix=release-tools [email protected]:<user>/csi-release-tools.git <my-new-or-existing-branch>` - push to a new branch before submitting a PR
+
+verify-shellcheck.sh
+--------------------
+
+The [verify-shellcheck.sh](./verify-shellcheck.sh) script in this repo
+is a stripped down copy of the [corresponding
+script](https://github.com/kubernetes/kubernetes/blob/release-1.14/hack/verify-shellcheck.sh)
+in the Kubernetes repository. It can be used to check for certain
+errors shell scripts, like missing quotation marks. The default
+`test-shellcheck` target in [build.make](./build.make) only checks the
+scripts in this directory. Components can add more directories to
+`TEST_SHELLCHECK_DIRS` to check also other scripts.
+
+End-to-end testing
+------------------
+
+A repo that wants to opt into testing via Prow must set up a top-level
+`.prow.sh`. Typically that will source `prow.sh` and then transfer
+control to it:
+
+``` bash
+#! /bin/bash -e
+
+. release-tools/prow.sh
+main
+```
+
+All Kubernetes-CSI repos are expected to switch to Prow. For details
+on what is enabled in Prow, see
+https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-csi
+
+Test results for periodic jobs are visible in
+https://testgrid.k8s.io/sig-storage-csi
+
+It is possible to reproduce the Prow testing locally on a suitable machine:
+- Linux host
+- Docker installed
+- code to be tested checkout out in `$GOPATH/src/<import path>`
+- `cd $GOPATH/src/<import path> && ./.prow.sh`
+
+Beware that the script intentionally doesn't clean up after itself and
+modifies the content of `$GOPATH`, in particular the `kubernetes` and
+`kind` repositories there. Better run it in an empty, disposable
+`$GOPATH`.
+
+When it terminates, the following command can be used to get access to
+the Kubernetes cluster that was brought up for testing (assuming that
+this step succeeded):
+
+    export KUBECONFIG="$(kind get kubeconfig-path --name="csi-prow")"
+
+It is possible to control the execution via environment variables. See
+`prow.sh` for details. Particularly useful is testing against different
+Kubernetes releases:
+
+    CSI_PROW_KUBERNETES_VERSION=1.13.3 ./.prow.sh
+    CSI_PROW_KUBERNETES_VERSION=latest ./.prow.sh

release-tools/build.make

@@ -98,7 +98,7 @@ test:
 test: test-go
 test-go:
 	@ echo; echo "### $@:"
-	go test `go list ./... | grep -v 'vendor' $(TEST_GO_FILTER_CMD)` $(TESTARGS)
+	go test `go list ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)
 
 .PHONY: test-vet
 test: test-vet
@@ -117,8 +117,37 @@ test-fmt:
 		false; \
 	fi
 
+# This test only runs when dep >= 0.5 is installed, which is the case for the CI setup.
+.PHONY: test-vendor
+test: test-vendor
+test-vendor:
+	@ echo; echo "### $@:"
+	@ case "$$(dep version 2>/dev/null | grep 'version *:')" in \
+		*v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \
+		*) echo "skipping check, dep >= 0.5 required";; \
+	esac
+
 .PHONY: test-subtree
 test: test-subtree
 test-subtree:
 	@ echo; echo "### $@:"
 	./release-tools/verify-subtree.sh release-tools
+
+# Components can extend the set of directories which must pass shellcheck.
+# The default is to check only the release-tools directory itself.
+TEST_SHELLCHECK_DIRS=release-tools
+.PHONY: test-shellcheck
+test: test-shellcheck
+test-shellcheck:
+	@ echo; echo "### $@:"
+	@ ret=0; \
+	if ! command -v docker; then \
+		echo "skipped, no Docker"; \
+		exit 0; \
+        fi; \
+	for dir in $(abspath $(TEST_SHELLCHECK_DIRS)); do \
+		echo; \
+		echo "$$dir:"; \
+		./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \
+	done; \
+	exit $$ret

release-tools/filter-junit.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ * This command filters a JUnit file such that only tests with a name
+ * matching a regular expression are passed through. By concatenating
+ * multiple input files it is possible to merge them into a single file.
+ */
+package main
+
+import (
+	"encoding/xml"
+	"flag"
+	"io/ioutil"
+	"os"
+	"regexp"
+)
+
+var (
+	output = flag.String("o", "-", "junit file to write, - for stdout")
+	tests  = flag.String("t", "", "regular expression matching the test names that are to be included in the output")
+)
+
+/*
+ * TestSuite represents a JUnit file. Due to how encoding/xml works, we have
+ * represent all fields that we want to be passed through. It's therefore
+ * not a complete solution, but good enough for Ginkgo + Spyglass.
+ */
+type TestSuite struct {
+	XMLName   string     `xml:"testsuite"`
+	TestCases []TestCase `xml:"testcase"`
+}
+
+type TestCase struct {
+	Name      string     `xml:"name,attr"`
+	Time      string     `xml:"time,attr"`
+	SystemOut string     `xml:"system-out,omitempty"`
+	Failure   string     `xml:"failure,omitempty"`
+	Skipped   SkipReason `xml:"skipped,omitempty"`
+}
+
+// SkipReason deals with the special <skipped></skipped>:
+// if present, we must re-encode it, even if empty.
+type SkipReason string
+
+func (s *SkipReason) UnmarshalText(text []byte) error {
+	*s = SkipReason(text)
+	if *s == "" {
+		*s = " "
+	}
+	return nil
+}
+
+func (s SkipReason) MarshalText() ([]byte, error) {
+	if s == " " {
+		return []byte{}, nil
+	}
+	return []byte(s), nil
+}
+
+func main() {
+	var junit TestSuite
+	var data []byte
+
+	flag.Parse()
+
+	re := regexp.MustCompile(*tests)
+
+	// Read all input files.
+	for _, input := range flag.Args() {
+		if input == "-" {
+			if _, err := os.Stdin.Read(data); err != nil {
+				panic(err)
+			}
+		} else {
+			var err error
+			data, err = ioutil.ReadFile(input)
+			if err != nil {
+				panic(err)
+			}
+		}
+		if err := xml.Unmarshal(data, &junit); err != nil {
+			panic(err)
+		}
+	}
+
+	// Keep only matching testcases. Testcases skipped in all test runs are only stored once.
+	filtered := map[string]TestCase{}
+	for _, testcase := range junit.TestCases {
+		if !re.MatchString(testcase.Name) {
+			continue
+		}
+		entry, ok := filtered[testcase.Name]
+		if !ok || // not present yet
+			entry.Skipped != "" && testcase.Skipped == "" { // replaced skipped test with real test run
+			filtered[testcase.Name] = testcase
+		}
+	}
+	junit.TestCases = nil
+	for _, testcase := range filtered {
+		junit.TestCases = append(junit.TestCases, testcase)
+	}
+
+	// Re-encode.
+	data, err := xml.MarshalIndent(junit, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+
+	// Write to output.
+	if *output == "-" {
+		if _, err := os.Stdout.Write(data); err != nil {
+			panic(err)
+		}
+	} else {
+		if err := ioutil.WriteFile(*output, data, 0644); err != nil {
+			panic(err)
+		}
+	}
+}