[Task] VX-189 Use bats to provide integration test.

.travis.yml

@@ -5,6 +5,8 @@ sudo: required
 services:
   - docker
 
+dist: xenial
+
 go:
   - "1.10.x"
 
@@ -17,8 +19,10 @@ notifications:
 
 before_install:
   - go get -u github.com/kardianos/govendor
-  - sudo apt-get install -y git build-essential openvswitch-switch socat
-  - docker run -d -v /run/openvswitch/db.sock:/var/run/openvswitch/db.sock -p 50051:50051 sdnvortex/network-controller:v0.3.0 /go/bin/server -tcp=0.0.0.0:50051
+  - sudo apt-get install -y git build-essential openvswitch-switch socat jq httpie
+  - sudo add-apt-repository ppa:duggan/bats --yes
+  - sudo apt-get update -qq
+  - sudo apt-get install -qq bats
 
 install:
   - make pre-build
@@ -28,6 +32,7 @@ before_script:
   - sudo mount --make-rshared /
     # Make /sys mounted as rshared to support cadvisor
   - sudo mount --make-rshared /sys
+  - sudo mount --make-rshared /var/run
     # # Download kubectl, which is a requirement for using minikube.
   - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.9.0/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
     # # Download minikube.
@@ -42,7 +47,7 @@ before_script:
     # # Download Helm.
   - curl -L https://storage.googleapis.com/kubernetes-helm/helm-v2.9.1-linux-amd64.tar.gz > helm-v2.9.1-linux-amd64.tar.gz && tar -zxvf helm-v2.9.1-linux-amd64.tar.gz && chmod +x linux-amd64/helm && sudo mv linux-amd64/helm /usr/local/bin/helm
     # # Install nsenter
-  - git clone https://github.com/jetstack/ubuntu-nsenter.git && docker build -t ubuntu-nsenter ubuntu-nsenter/ && docker run -v /usr/local/bin:/hostbin ubuntu-nsenter cp /nsenter /hostbin/nsenter
+  - docker run -v /usr/local/bin:/hostbin hwchiu/ubuntu-nsenter cp /nsenter /hostbin/nsenter
 
 script:
   - kubectl cluster-info
@@ -61,8 +66,12 @@ script:
   - make apps.launch-apps
   # # Check if prometheus is ready
   - until curl --connect-timeout 1 -sL -w "%{http_code}\\n" http://`kubectl get service -n vortex prometheus -o jsonpath="{.spec.clusterIP}"`:9090/api/v1/query?query=prometheus_build_info -o /dev/null | grep 200; do sleep 1; echo "wait the prometheus to be available"; kubectl get pods --all-namespaces; done
+  - until [ `kubectl  -n vortex get --no-headers pods | awk '{c[$3]++}END{ print NR-c["Running"]}'` -eq 0 ]; do sleep 1; echo "wait all pod running"; kubectl -n vortex get pods;  done
+  - kubectl -n vortex describe daemonset/network-controller-server-tcp
+  - kubectl -n vortex describe daemonset/network-controller-server-unix
   # # Start to test and produce the coverage report
   - sudo -E PATH=$PATH TEST_GRPC=1 TEST_PROMETHEUS=1 make src.test-coverage-minikube
+  - make src.test-bats
   # # Build the docker image if need it
   # - if [ "$TRAVIS_BRANCH" == "develop" ]; then docker build --tag sdnvortex/vortex --file ./dockerfiles/Dockerfile . ; fi
 

Makefile

@@ -75,6 +75,11 @@ src.test-coverage-vagrant:
 
 ## check build env #############################
 
+.PHONY: src.test-bats
+src.test-bats:
+	cd tests; \
+	bats .;
+
 .PHONY: check-govendor
 check-govendor:
 	$(info check govendor)
@@ -91,13 +96,12 @@ apps.init-helm:
 
 .PHONY: apps.launch-apps
 apps.launch-apps:
-	helm install --name foundation --debug --wait --set global.environment=local deploy/helm/foundation
-	helm install --name prometheus --debug --wait --set global.environment=local deploy/helm/apps/charts/prometheus
-	helm install --name vortex-server --debug --wait --set global.environment=local deploy/helm/apps/charts/vortex-server
+	helm install --name vortex-foundation --debug --set global.environment=testing deploy/helm/foundation
+	helm install --name vortex-apps --debug --set global.environment=testing deploy/helm/apps/
 
 .PHONY: apps.teardown
 apps.teardown:
-	helm ls --short | xargs -L1 helm delete
+	helm ls --short | xargs -i helm delete --purge {}
 
 ## dockerfiles/ ########################################
 

deploy/helm/foundation/charts/rbac/templates/role.yaml

@@ -1,6 +1,6 @@
 ---
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: vortex-admin
 rules:
@@ -16,14 +16,15 @@ rules:
     - '*'
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: vortex-admin
+  namespace: vortex
 roleRef:
-  apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: vortex-admin
+  apiGroup: rbac.authorization.k8s.io
 subjects:
 - kind: ServiceAccount
-  name: vortex
+  name: vortex-admin
   namespace: vortex

developers/Vagrantfile

@@ -18,7 +18,7 @@ echo "=== Install Vortex Server Environment ==="
 sudo mkdir -p "/etc/systemd/system/docker.service.d/"
 sudo cp "/tmp/override.conf" "/etc/systemd/system/docker.service.d/override.conf"
 sudo apt-get update
-sudo apt-get install -y git build-essential nfs-common ntp
+sudo apt-get install -y git build-essential nfs-common ntp jq bats httpie
 
 # Install Docker
 # kubernetes official max validated version: 17.03.2~ce-0~ubuntu-xenial
@@ -79,6 +79,9 @@ export PATH=/home/$USER/protoc/bin:$PATH:$GOROOT/bin:$GOPATH/bin
 # setup golang dir
 mkdir -p /home/$USER/go/src
 rm -rf /home/$USER/go1.10.2.linux-amd64.tar.gz
+sudo add-apt-repository ppa:duggan/bats --yes
+sudo apt-get update -qq
+sudo apt-get install -qq bats
 
 # Install protoc
 # Env for proto

tests/00-env.bats

@@ -0,0 +1,27 @@
+#!/usr/bin/env bats
+
+@test "Check the httpie" {
+    run which http
+    [ $status = 0 ]
+}
+
+@test "Check the jq" {
+    run which jq
+    [ $status = 0 ]
+}
+
+@test "Check the OpenvSwitch" {
+    run which ovs-vsctl
+    [ $status = 0 ]
+}
+
+@test "Check kubernetes tools" {
+    run which kubectl
+    [ $status = 0 ]
+}
+
+@test "Check kubernetes cluster" {
+    run kubectl get nodes
+    [ $status = 0 ]
+    [[ ${lines[0]} != "No resources found." ]]
+}

tests/01-function.bats

@@ -0,0 +1,58 @@
+#!/usr/bin/env bats
+
+load data
+setup() {
+    sed -i "s/@NODENAME@/${nodeName}/" networks.json
+    sed -i "s/@NETWORKNAME@/${networkName}/" networks.json
+    sed -i "s/@NETWORKNAME@/${networkName}/" pod.json
+    sed -i "s/@PODNAME@/${podName}/" pod.json
+}
+
+teardown() {
+    git checkout networks.json
+    git checkout pod.json
+}
+
+@test "Create network" {
+    http -v --check-status 127.0.0.1:32326/v1/networks < networks.json
+    [ $? = 0 ]
+}
+
+@test "List network" {
+    run bash -c "http http://127.0.0.1:32326/v1/networks/ 2>/dev/null | jq -r '.[] | select(.name == \"${networkName}\").name'"
+    [ "$output" = "${networkName}" ]
+    [ $status = 0 ]
+}
+
+@test "Create Pod" {
+    http -v --check-status 127.0.0.1:32326/v1/pods < pod.json
+    [ $? = 0 ]
+    #Wait the Pod
+    #jsonpath="{.status.phase}"
+    NEXT_WAIT_TIME=0
+    WAIT_LIMIT=40
+    until kubectl get pods ${podName} -o jsonpath="{.status.phase}" | grep "Running" || [ $NEXT_WAIT_TIME -eq $WAIT_LIMIT ]; do
+       sleep 2
+       kubectl get pods ${podName}
+       NEXT_WAIT_TIME=$((NEXT_WAIT_TIME+ 1))
+    done
+    [ $NEXT_WAIT_TIME != $WAIT_LIMIT ]
+}
+
+@test "List Pod" {
+   run bash -c "http http://127.0.0.1:32326/v1/pods/ 2>/dev/null | jq -r '.[] | select(.name == \"${podName}\").name'"
+   [ "$output" = "${podName}" ]
+   [ $status = 0 ]
+}
+
+@test "Delete Pod" {
+    run bash -c 'http http://127.0.0.1:32326/v1/pods/ 2>/dev/null | jq -r ".[0].id"'
+    run http DELETE http://127.0.0.1:32326/v1/pods/${output} 2>/dev/null
+    [ $status = 0 ]
+}
+
+@test "Delete Network" {
+    run bash -c 'http http://127.0.0.1:32326/v1/networks/ 2>/dev/null | jq -r ".[0].id"'
+    run http DELETE http://127.0.0.1:32326/v1/networks/${output} 2>/dev/null
+    [ $status = 0 ]
+}

tests/data.bash

@@ -0,0 +1,6 @@
+nodeName=`kubectl get nodes | grep "Ready" | awk '{print $1}'`
+if [ -z "$networkName" ]; then
+    export name=$(date | md5sum | cut -b 1-19)
+    export podName="test-pod-$name"
+    export networkName="test-network-$name"
+fi

tests/networks.json

@@ -0,0 +1,15 @@
+{
+	"type":"system",
+	"isDPDKPort":false,
+	"name":"@NETWORKNAME@",
+	"vlanTags":[],
+	"bridgeName":"br0",
+	"nodes":[
+		{	
+			"name": "@NODENAME@",
+			"physicalInterfaces": [{
+				"name":"eth10"	
+			}]
+		}
+	]
+}

tests/pod.json

@@ -0,0 +1,38 @@
+{
+	"name": "@PODNAME@",
+	"labels":{},
+	"namespace":"default",
+	"containers":[
+		{
+			"name":"first-container",
+			"image":"busybox",
+            "command":["sleep","3600"]
+		}
+	],
+	"networks":[
+		{
+			"name":"@NETWORKNAME@",
+			"ifName":"eth12",
+			"vlan":0,
+			"ipAddress":"1.2.3.4",
+			"netmask":"255.255.255.0",
+			"vlanTag":123
+		},
+				{
+			"name":"@NETWORKNAME@",
+			"ifName":"eth13",
+			"vlan":0,
+			"ipAddress":"1.2.4.5",
+			"netmask":"255.255.255.0",
+			"vlanTag":124
+		},
+				{
+			"name":"@NETWORKNAME@",
+			"ifName":"eth15",
+			"vlan":0,
+			"ipAddress":"15.2.4.5",
+			"netmask":"255.255.255.0"
+		}	
+	],
+	"volumes":[]
+}