DEV VX-185: Using helm deploy prometheus and mongodb in Jenkins and Travis

.gitignore

@@ -12,6 +12,10 @@ vagrant/credentials
 **/*.vmdk
 **/*.creds
 
+# helm
+helm/**/*.tgz
+helm/**/requirements.lock
+
 # ansible
 **/*.retry
 

.travis.yml

@@ -3,7 +3,6 @@ language: go
 sudo: required
 
 services:
-  - mongodb
   - docker
 
 go:
@@ -18,7 +17,7 @@ notifications:
 
 before_install:
   - go get -u github.com/kardianos/govendor
-  - sudo apt-get install -y git build-essential openvswitch-switch
+  - sudo apt-get install -y git build-essential openvswitch-switch socat
   - docker run -d -v /run/openvswitch/db.sock:/var/run/openvswitch/db.sock -p 50051:50051 sdnvortex/network-controller:v0.2.1 /go/bin/server -tcp=0.0.0.0:50051
 
 install:
@@ -33,11 +32,17 @@ before_script:
   - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.9.0/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
     # # Download minikube.
   - curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
-  - sudo minikube start --vm-driver=none --bootstrapper=localkube --kubernetes-version=v1.9.0
+  - sudo minikube start --vm-driver=none --bootstrapper=localkube --kubernetes-version=v1.9.0 --extra-config=apiserver.Authorization.Mode=RBAC
     # # Fix the kubectl context, as it's often stale.
   - minikube update-context
     # # Wait for Kubernetes to be up and ready.
   - JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done
+    # # Enable rbac.
+  - kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default
+    # # Download Helm.
+  - curl -L https://storage.googleapis.com/kubernetes-helm/helm-v2.9.1-linux-amd64.tar.gz > helm-v2.9.1-linux-amd64.tar.gz && tar -zxvf helm-v2.9.1-linux-amd64.tar.gz && chmod +x linux-amd64/helm && sudo mv linux-amd64/helm /usr/local/bin/helm
+    # # Install nsenter
+  - git clone https://github.com/jetstack/ubuntu-nsenter.git && docker build -t ubuntu-nsenter ubuntu-nsenter/ && docker run -v /usr/local/bin:/hostbin ubuntu-nsenter cp /nsenter /hostbin/nsenter
 
 script:
   - kubectl cluster-info
@@ -46,13 +51,19 @@ script:
   - JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lcomponent=kube-addon-manager -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1;echo "waiting for kube-addon-manager to be available"; kubectl get pods --all-namespaces; done
   # # Wait for kube-dns to be ready.
   - JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lk8s-app=kube-dns -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1;echo "waiting for kube-dns to be available"; kubectl get pods --all-namespaces; done
-  - kubectl apply -f deploy/kubernetes/apps/monitoring/ -R
-  # Chck the functrion of prometheus
-  - until curl --connect-timeout 1 -sL -w "%{http_code}\\n" http://`kubectl get service -n monitoring prometheus -o jsonpath="{.spec.clusterIP}"`:9090/api/v1/query?query=prometheus_build_info -o /dev/null | grep 200; do sleep 1; echo "wait the prometheus to be available"; done
-  # Wait the cadvisor to be ready
-  - JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n monitoring get pods -lname=cadvisor -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1;echo "waiting for cadvisor to be available"; kubectl get pods --all-namespaces; done
   - make build
-  - sudo -E PATH=$PATH TEST_GRPC=1 TEST_PROMETHEUS=1 make src.test-coverage
+  # # Initial helm
+  - make apps.init-helm
+  # # Check if tiller is ready
+  - JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lname=tiller -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; echo "wait the tiller to be available"; done
+  - helm version
+  # # Launch apps (mongodb, prometheus)
+  - make apps.launch-apps
+  # # Check if prometheus is ready
+  - until curl --connect-timeout 1 -sL -w "%{http_code}\\n" http://`kubectl get service -n monitoring prometheus -o jsonpath="{.spec.clusterIP}"`:9090/api/v1/query?query=prometheus_build_info -o /dev/null | grep 200; do sleep 1; echo "wait the prometheus to be available"; done
+  # # Start to test and produce the coverage report
+  - sudo -E PATH=$PATH TEST_GRPC=1 TEST_PROMETHEUS=1 make src.test-coverage-minikube
+  # # Build the docker image if need it
   - if [ "$TRAVIS_BRANCH" == "develop" ]; then docker build --tag sdnvortex/vortex --file ./dockerfiles/Dockerfile . ; fi
 
 before_deploy:

Jenkinsfile

@@ -1,4 +1,4 @@
-pipeline {
+  pipeline {
     agent {
         dockerfile {
             dir "src/github.com/linkernetworks/vortex/jenkins"
@@ -64,8 +64,16 @@ pipeline {
             steps {
                 withEnv(["GOPATH+AA=${env.WORKSPACE}", "TEST_PROMETHEUS=1"]) {
                     dir ("src/github.com/linkernetworks/vortex") {
-                        sh "make src.test-prometheus 2>&1 | tee >(go-junit-report > report.xml)"
-                        sh "make src.test-coverage 2>&1 | tee >(go-junit-report > report.xml)"
+                        waitUntil {
+                            fileExists "ready"
+                        }
+                        sh "make apps.init-helm 2>&1 | tee >(go-junit-report > report.xml)"
+	                    sh "JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lname=tiller -o jsonpath=\"\$JSONPATH\" 2>&1 | grep -q \"Ready=True\"; do sleep 1; echo \"wait the tiller to be available\"; done"
+
+                        sh "make apps.launch-apps 2>&1 | tee >(go-junit-report > report.xml)"
+                        sh "until curl --connect-timeout 1 -sL -w \"%{http_code}\\n\" http://`kubectl get service -n monitoring prometheus -o jsonpath=\"{.spec.clusterIP}\"`:9090/api/v1/query?query=prometheus_build_info -o /dev/null | grep 200; do sleep 1; echo \"wait the prometheus to be available\"; done"
+
+                        sh "make src.test-coverage-minikube 2>&1 | tee >(go-junit-report > report.xml)"
                         junit "report.xml"
                         sh 'gocover-cobertura < build/src/coverage.txt > cobertura.xml'
                         cobertura coberturaReportFile: "cobertura.xml", failNoReports: true, failUnstable: true

Makefile

@@ -53,33 +53,49 @@ src.test:
 src.install:
 	$(GO) install -v ./src/...
 
-.PHONY: src.test-prometheus
-src.test-prometheus:
-	kubectl apply -f deploy/kubernetes/apps/monitoring/ -R
-	JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done
-	#Verify kube-addon-manager.
-	#kube-addon-manager is responsible for managing other kubernetes components, such as kube-dns, dashboard, storage-provisioner..
-	JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lcomponent=kube-addon-manager -o jsonpath="$$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1;echo "waiting for kube-addon-manager to be available"; kubectl get pods --all-namespaces; done
-	# Wait for kube-dns to be ready.
-	JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lk8s-app=kube-dns -o jsonpath="$$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1;echo "waiting for kube-dns to be available"; kubectl get pods --all-namespaces; done
-	#Chck the functrion of prometheus
-	until curl --connect-timeout 1 -sL -w "%{http_code}\\n" http://`kubectl get service -n monitoring prometheus -o jsonpath="{.spec.clusterIP}"`:9090/api/v1/query?query=prometheus_build_info -o /dev/null | grep 200; do sleep 1; echo "wait the prometheus to be available"; done
-	#Wait the cadvisor to be ready
-	JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n monitoring get pods -lname=cadvisor -o jsonpath="$$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1;echo "waiting for cadvisor to be available"; kubectl get pods --all-namespaces; done
-
 .PHONY: src.test-coverage
 src.test-coverage:
 	$(MKDIR_P) $(BUILD_FOLDER)/src/
 	$(GO) test -v -race -coverprofile=$(BUILD_FOLDER)/src/coverage.txt -covermode=atomic ./src/...
 	$(GO) tool cover -html=$(BUILD_FOLDER)/src/coverage.txt -o $(BUILD_FOLDER)/src/coverage.html
 
+.PHONY: src.test-coverage-minikube
+src.test-coverage-minikube:
+	sed -i.bak "s/localhost:9090/$$(minikube ip):30003/g; s/localhost:27017/$$(minikube ip):31717/g" config/testing.json
+	$(MAKE) src.test-coverage
+	mv config/testing.json.bak config/testing.json
+
+.PHONY: src.test-coverage-vagrant
+src.test-coverage-vagrant:
+	sed -i.bak "s/localhost:9090/172.17.8.100:30003/g; s/localhost:27017/172.17.8.100:31717/g" config/testing.json
+	$(MAKE) src.test-coverage
+	mv config/testing.json.bak config/testing.json
+
 ## check build env #############################
 
 .PHONY: check-govendor
 check-govendor:
 	$(info check govendor)
 	@[ "`which $(GO_VENDOR)`" != "" ] || (echo "$(GO_VENDOR) is missing"; false)
 
+## launch apps #############################
+
+.PHONY: apps.init-helm
+apps.init-helm:
+	helm init
+	kubectl create serviceaccount --namespace kube-system tiller
+	kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
+	kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}'
+
+.PHONY: apps.launch-apps
+apps.launch-apps:
+	helm install --debug --wait --set global.environment=testing deploy/helm/foundation/mongodb
+	helm install --debug --wait --set global.environment=testing deploy/helm/apps/prometheus/
+
+.PHONY: apps.teardown
+apps.teardown:
+	helm ls --short | xargs -L1 helm delete
+
 ## dockerfiles/ ########################################
 
 .PHONY: dockerfiles.build

deploy/helm/apps/prometheus/templates/prometheus-svc.yaml

@@ -11,7 +11,7 @@ spec:
   ports:
   - port: {{ .Values.service.ports.port }}
     targetPort: {{ .Values.service.ports.targetPort }}
-  {{ else if eq .Values.global.environment "local"}}
+  {{ else if or (eq .Values.global.environment "local") (eq .Values.global.environment "testing") }}
   type: NodePort
   ports:
   - port: {{ .Values.service.ports.port }}

deploy/helm/apps/prometheus/values.yaml

@@ -10,7 +10,7 @@ service:
   ports:
     port: 9090
     targetPort: 9090
-    nodePort: 32326
+    nodePort: 30003
 
 
 # dependenvies

deploy/helm/foundation/mongodb/charts/hostpath/templates/manual.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   storageClassName: manual
   capacity:
-    storage: {{ .Values.capacity.storage }}
+    storage: {{ .Values.global.storage }}
   accessModes:
     - ReadWriteOnce
   hostPath:

deploy/helm/foundation/mongodb/charts/hostpath/values.yaml

@@ -1,2 +1,2 @@
 capacity:
-  storage: 10Gi
+  storage: 1Gi

deploy/helm/foundation/mongodb/templates/service.yaml

@@ -4,12 +4,12 @@ kind: Service
 metadata:
   name: mongo
 spec:
-  {{ if eq .Values.environment "develop" }}
+  {{ if eq .Values.global.environment "develop" }}
   type: ClusterIP
   ports:
   - port: {{ .Values.service.ports.port }} 
     targetPort: {{ .Values.service.ports.targetPort }}
-  {{ else if eq .Values.environment "local"}}
+  {{ else if or (eq .Values.global.environment "local") (eq .Values.global.environment "testing") }}
   type: NodePort
   ports:
   - port: {{ .Values.service.ports.port }}

deploy/helm/foundation/mongodb/templates/statefulset.yaml

@@ -31,13 +31,13 @@ spec:
   - metadata:
       name: mongodb
     spec:
-      {{ if eq .Values.environment "develop" }}
+      {{ if eq .Values.global.environment  "develop" }}
       accessModes: [ "ReadWriteMany" ]
       storageClassName: "slow-many"
-      {{ else if eq .Values.environment "local"}}
+      {{ else if or (eq .Values.global.environment "local") (eq .Values.global.environment "testing") }}
       accessModes: [ "ReadWriteOnce" ]
       storageClassName: "manual"
       {{ end }}
       resources:
         requests:
-          storage: 10Gi
+          storage: {{ .Values.global.storage }}

deploy/helm/foundation/mongodb/values.yaml

@@ -1,4 +1,6 @@
-environment: develop
+global:
+  environment: develop
+  storage: 1Gi
 
 hostpath:
   enabled: false

deploy/kubernetes/apps/monitoring/prometheus/prometheus-deployment.yaml

@@ -12,6 +12,10 @@ spec:
       labels:
         app: prometheus
     spec:
+      initContainers:
+      - name: init-kube-state-metricsks
+        image: busybox
+        command: ["sh", "-c", "until nslookup kube-state-metrics.monitoring.svc.cluster.local; do echo waiting for kube-state-metricsks; sleep 2; done;"]
       containers:
       - image: prom/prometheus:v2.2.1
         name: prometheus

jenkins/Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:16.04
 
-RUN apt-get update && apt-get install -y curl software-properties-common git
+RUN apt-get update && apt-get install -y curl software-properties-common git socat
 RUN curl -fsSL get.docker.com | sh
 
 ##
@@ -20,13 +20,18 @@ RUN go get -u github.com/jstemmer/go-junit-report
 RUN go get -u github.com/t-yuki/gocover-cobertura
 
 ##
-## MongoDB
+## kubectl
 ##
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
-RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | \
-    tee /etc/apt/sources.list.d/mongodb-org-3.6.list
-RUN apt-get update
-RUN apt-get install -y mongodb-org
+RUN curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.9.0/bin/linux/amd64/kubectl
+RUN chmod +x kubectl
+RUN mv kubectl /usr/local/bin/
+
+##
+## Helm
+##
+RUN curl -L https://storage.googleapis.com/kubernetes-helm/helm-v2.9.1-linux-amd64.tar.gz > helm-v2.9.1-linux-amd64.tar.gz
+RUN tar -zxvf helm-v2.9.1-linux-amd64.tar.gz
+RUN mv linux-amd64/helm /usr/local/bin/helm
 
 ##
 ## Make jenkins can do anything

jenkins/entry.sh

@@ -15,12 +15,15 @@ sudo chown -R jenkins:jenkins /home/jenkins
 sudo mount --make-rshared /
 # Make /sys mounted as rshared to support cadvisor
 sudo mount --make-rshared /sys
-# # Download kubectl, which is a requirement for using minikube.
-curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.9.0/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
 # Download minikube.
 curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
-sudo env CHANGE_MINIKUBE_NONE_USER=true minikube start --vm-driver=none --bootstrapper=localkube --kubernetes-version=v1.9.0
+sudo env CHANGE_MINIKUBE_NONE_USER=true minikube start --vm-driver=none --bootstrapper=localkube --kubernetes-version=v1.9.0 --extra-config=apiserver.Authorization.Mode=RBAC
 # Fix the kubectl context, as it's often stale.
 minikube update-context
+# Enable rbac
+kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default
+
+# Check if entry.sh finish or not
+touch src/github.com/linkernetworks/vortex/ready
 
 bash

src/prometheuscontroller/expression_test.go

@@ -22,7 +22,7 @@ type PrometheusExpressionTestSuite struct {
 }
 
 func (suite *PrometheusExpressionTestSuite) SetupSuite() {
-	cf := config.MustRead("../../config/local.json")
+	cf := config.MustRead("../../config/testing.json")
 	suite.sp = serviceprovider.New(cf)
 }
 

src/prometheuscontroller/query_test.go

@@ -2,14 +2,15 @@ package prometheuscontroller
 
 import (
 	"fmt"
-	"github.com/linkernetworks/vortex/src/config"
-	"github.com/linkernetworks/vortex/src/serviceprovider"
-	"github.com/prometheus/common/model"
-	"github.com/stretchr/testify/suite"
 	"math/rand"
 	"os"
 	"testing"
 	"time"
+
+	"github.com/linkernetworks/vortex/src/config"
+	"github.com/linkernetworks/vortex/src/serviceprovider"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/suite"
 )
 
 func init() {
@@ -23,7 +24,7 @@ type PrometheusQueryTestSuite struct {
 }
 
 func (suite *PrometheusQueryTestSuite) SetupSuite() {
-	cf := config.MustRead("../../config/local.json")
+	cf := config.MustRead("../../config/testing.json")
 	suite.sp = serviceprovider.New(cf)
 	suite.containerName = "cadvisor"
 }