Add CA generation and mount it to /ca in containers (#28)

* Add CA generation

* Update internal/docker/builder.go

* Update CA volume mounting logic.

* Update CA cert creation process

Make sure that the certs are created at test start.

* Fix grammar in some comments

* Gate Complement PKI using COMPLEMENT_CA.

Set COMPLEMENT_CA=true to enable Complement PKI

* Update README.md

Co-authored-by: Kegsay <[email protected]>

README.md

@@ -32,11 +32,37 @@ $ COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -timeout 30s -run '^(
 - The homeserver needs to `200 OK` requests to `GET /_matrix/client/versions`.
 - The homeserver needs to manage its own storage within the image.
 - The homeserver needs to accept the server name given by the environment variable `SERVER_NAME` at runtime.
+- The homeserver can use the CA certificate mounted at /ca to create its own TLS cert (see [Complement PKI](README.md#complement-pki)).
 
 #### Why 'Complement'?
 
 Because **M**<sup>*C*</sup> = **1** - **M**
 
+#### Complement PKI
+
+As the Matrix federation protocol expects federation endpoints to be served with valid TLS certs,
+Complement will create a self-signed CA cert to use for creating valid TLS certs in homeserver containers.
+
+To enable it pass `COMPLEMENT_CA=true` to complement or the docker container.
+If not used, the homeserver needs to not validate the cert when federating.
+To check whether complements runs in PKI mode, `COMPLEMENT_CA` is passed through to the homeserver containers.
+
+The public key to add to the trusted cert store (e.g., /etc/ca-certificates) is mounted at: `/ca/ca.crt`
+The private key to sign the created TLS cert is mounted at: `/ca/ca.key`
+
+For example, to sign your certificate for the homeserver, run at each container start (Ubuntu):
+```
+openssl genrsa -out $SERVER_NAME.key 2048
+openssl req -new -sha256 -key $SERVER_NAME.key -subj "/C=US/ST=CA/O=MyOrg, Inc./CN=$SERVER_NAME" -out $SERVER_NAME.csr
+openssl x509 -req -in $SERVER_NAME.csr -CA /ca/ca.crt -CAkey /ca/ca.key -CAcreateserial -out $SERVER_NAME.crt -days 1 -sha256
+```
+
+To add the CA cert to your trust store (Ubuntu):
+```
+cp /root/ca.crt /usr/local/share/ca-certificates/complement.crt
+update-ca-certificates
+```
+
 #### Sytest parity
 
 ```

dockerfiles/ComplementCIBuildkite.Dockerfile

@@ -7,4 +7,6 @@
 FROM golang:1.15-buster
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh
 ADD https://github.com/matrix-org/complement/archive/master.tar.gz .
-RUN tar -xzf master.tar.gz && cd complement-master && go mod download 
+RUN tar -xzf master.tar.gz && cd complement-master && go mod download 
+
+VOLUME [ "/ca" ]

internal/docker/builder.go

@@ -15,10 +15,13 @@ package docker
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log"
+	"io/ioutil"
 	"net/http"
 	"os"
+	"path"
 	"runtime"
 	"strings"
 	"sync"
@@ -27,6 +30,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
 	client "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
@@ -291,27 +295,115 @@ func (d *Builder) deployBaseImage(blueprintName, hsName, contextStr, networkID s
 	return deployImage(d.Docker, d.BaseImage, d.CSAPIPort, fmt.Sprintf("complement_%s", contextStr), blueprintName, hsName, contextStr, networkID)
 }
 
+// getCaVolume returns the correct mounts and volumes for providing a CA to homeserver containers.
+func getCaVolume(docker *client.Client, ctx context.Context) (map[string]struct{}, []mount.Mount, error) {
+	var caVolume map[string]struct{}
+	var caMount []mount.Mount
+
+	if os.Getenv("CI") == "true" {
+		// When in CI, Complement itself is a container with the CA volume mounted at /ca.
+		// We need to mount this volume to all homeserver containers to synchronize the CA cert.
+		// This is needed to establish trust among all containers.
+
+		// Get volume mounted at /ca. First we get the container ID
+		// /proc/1/cpuset should be /docker/<containerId>
+		cpuset, err := ioutil.ReadFile("/proc/1/cpuset")
+		if err != nil {
+			return nil, nil, err
+		}
+		if !strings.Contains(string(cpuset), "docker") {
+			return nil, nil, errors.New("Could not identify container ID using /proc/1/cpuset")
+		}
+		cpusetList := strings.Split(strings.TrimSpace(string(cpuset)), "/")
+		containerId := cpusetList[len(cpusetList)-1]
+		container, err := docker.ContainerInspect(ctx, containerId)
+		if err != nil {
+			return nil, nil, err
+		}
+		// Get the volume that matches the destination in our complement container
+		var volumeName string
+		for i := range container.Mounts {
+			if container.Mounts[i].Destination == "/ca" {
+				volumeName = container.Mounts[i].Name
+			}
+		}
+		if volumeName == "" {
+			// We did not find a volume. This container might be created without a volume,
+			// or CI=true is passed but we are not running in a container.
+			// todo: log that we do not provide a CA volume mount?
+			return nil, nil, nil
+		} else {
+			caVolume = map[string]struct{}{
+				"/ca": {},
+			}
+			caMount = []mount.Mount{
+				{
+					Type:   mount.TypeVolume,
+					Source: volumeName,
+					Target: "/ca",
+				},
+			}
+		}
+	} else {
+		// When not in CI, our CA cert is placed in the current working dir.
+		// We bind mount this directory to all homeserver containers.
+		cwd, err := os.Getwd()
+		if err != nil {
+			return nil, nil, err
+		}
+		caCertificateDirHost := path.Join(cwd, "ca")
+		if _, err := os.Stat(caCertificateDirHost); os.IsNotExist(err) {
+			err = os.Mkdir(caCertificateDirHost, 0770)
+			if err != nil {
+				return nil, nil, err
+			}
+		}
+		caMount = []mount.Mount{
+			{
+				Type:   mount.TypeBind,
+				Source: path.Join(cwd, "ca"),
+				Target: "/ca",
+			},
+		}
+	}
+	return caVolume, caMount, nil
+}
+
 func deployImage(docker *client.Client, imageID string, csPort int, containerName, blueprintName, hsName, contextStr, networkID string) (*HomeserverDeployment, error) {
 	ctx := context.Background()
 	var extraHosts []string
+	var caVolume map[string]struct{}
+	var caMount []mount.Mount
+	var err error
+
 	if runtime.GOOS == "linux" {
 		// By default docker for linux does not expose this, so do it now.
 		// When https://github.com/moby/moby/pull/40007 lands in Docker 20, we should
 		// change this to be  `host.docker.internal:host-gateway`
 		extraHosts = []string{HostnameRunningComplement + ":172.17.0.1"}
 	}
+
+	if os.Getenv("COMPLEMENT_CA") == "true" {
+		caVolume, caMount, err = getCaVolume(docker, ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	body, err := docker.ContainerCreate(ctx, &container.Config{
 		Image: imageID,
-		Env:   []string{"SERVER_NAME=" + hsName},
+		Env:   []string{"SERVER_NAME=" + hsName, "COMPLEMENT_CA=" + os.Getenv("COMPLEMENT_CA")},
 		//Cmd:   d.ImageArgs,
 		Labels: map[string]string{
 			complementLabel:        contextStr,
 			"complement_blueprint": blueprintName,
 			"complement_hs_name":   hsName,
 		},
+		Volumes: caVolume,
 	}, &container.HostConfig{
 		PublishAllPorts: true,
 		ExtraHosts:      extraHosts,
+		Mounts:          caMount,
 	}, &network.NetworkingConfig{
 		map[string]*network.EndpointSettings{
 			hsName: {

internal/federation/server.go

@@ -9,7 +9,9 @@ import (
 	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"fmt"
+	"io/ioutil"
 	"math/big"
 	"net/http"
 	"os"
@@ -202,8 +204,119 @@ func (s *Server) Listen() (cancel func()) {
 	}
 }
 
+// Get or create local CA cert. This is used to create the federation TLS cert.
+// In addition, it is passed to homeserver containers to create TLS certs 
+// for the homeservers
+// This basically acts as a test only valid PKI.
+func GetOrCreateCaCert() (*x509.Certificate, *rsa.PrivateKey, error) {
+	var tlsCACertPath, tlsCAKeyPath string
+	if os.Getenv("CI") == "true" {
+		// When in CI we create the cert dir in the root directory instead.
+		tlsCACertPath = path.Join("/ca", "ca.crt")
+		tlsCAKeyPath = path.Join("/ca", "ca.key")
+	} else {
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, nil, err
+		}
+		tlsCACertPath = path.Join(wd, "ca", "ca.crt")
+		tlsCAKeyPath = path.Join(wd,"ca", "ca.key")
+		if _, err := os.Stat(path.Join(wd, "ca")); os.IsNotExist(err) {
+			err = os.Mkdir(path.Join(wd, "ca"), 0770)
+			if err != nil {
+				return nil, nil, err
+			}
+		}
+	}
+	if _, err := os.Stat(tlsCACertPath); err == nil {
+		if _, err := os.Stat(tlsCAKeyPath); err == nil {
+			// We already created a CA cert, let's use that.
+			dat, err := ioutil.ReadFile(tlsCACertPath)
+			if err != nil {
+				return nil, nil, err
+			}
+			block, _ := pem.Decode([]byte(dat))
+			if block == nil || block.Type != "CERTIFICATE" {
+				return nil, nil, errors.New("ca.crt is not a valid pem encoded x509 cert")
+			}
+			caCerts, err := x509.ParseCertificates(block.Bytes)
+			if err != nil {
+				return nil, nil, err
+			}
+			if len(caCerts) != 1 {
+				return nil, nil, errors.New("ca.crt contains none or more than one cert")
+			}
+			caCert := caCerts[0]
+			dat, err = ioutil.ReadFile(tlsCAKeyPath)
+			if err != nil {
+				return nil, nil, err
+			}
+			block, _ = pem.Decode([]byte(dat))
+			if block == nil || block.Type != "RSA PRIVATE KEY" {
+				return nil, nil, errors.New("ca.key is not a valid pem encoded rsa private key")
+			}
+			priv, err := x509.ParsePKCS1PrivateKey(block.Bytes)
+			if err != nil {
+				return nil, nil, err
+			}
+			return caCert, priv, nil
+		}
+	}
+
+	certificateDuration := time.Hour * 5
+	priv, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, err
+	}
+	notBefore := time.Now()
+	notAfter := notBefore.Add(certificateDuration)
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return nil, nil, err
+	}
+	caCert := x509.Certificate{
+		SerialNumber:          serialNumber,
+		NotBefore:             notBefore,
+		NotAfter:              notAfter,
+		IsCA:                  true,
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature | x509.KeyUsageCRLSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &caCert, &caCert, &priv.PublicKey, priv)
+	if err != nil {
+		return nil, nil, err
+	}
+	certOut, err := os.Create(tlsCACertPath)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	defer certOut.Close() // nolint: errcheck
+	if err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		return nil, nil, err
+	}
+
+	keyOut, err := os.OpenFile(tlsCAKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer keyOut.Close() // nolint: errcheck
+	err = pem.Encode(keyOut, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(priv),
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	return &caCert, priv, nil
+}
+
 // federationServer creates a federation server with the given handler
 func federationServer(name string, h http.Handler) (*http.Server, string, string, error) {
+	var derBytes []byte
 	srv := &http.Server{
 		Addr:    ":8448",
 		Handler: h,
@@ -231,10 +344,23 @@ func federationServer(name string, h http.Handler) (*http.Server, string, string
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		BasicConstraintsValid: true,
 	}
-	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-	if err != nil {
-		return nil, "", "", err
+	if os.Getenv("COMPLEMENT_CA") == "true" {
+		// Gate COMPLEMENT_CA
+		ca, caPrivKey, err := GetOrCreateCaCert()
+		if err != nil {
+			return nil, "", "", err
+		}
+		derBytes, err = x509.CreateCertificate(rand.Reader, &template, ca, &priv.PublicKey, caPrivKey)
+		if err != nil {
+			return nil, "", "", err
+		}
+	} else {
+		derBytes, err = x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+		if err != nil {
+			return nil, "", "", err
+		}
 	}
+
 	certOut, err := os.Create(tlsCertPath)
 	if err != nil {
 		return nil, "", "", err

tests/main_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/matrix-org/complement/internal/b"
 	"github.com/matrix-org/complement/internal/config"
 	"github.com/matrix-org/complement/internal/docker"
+	"github.com/matrix-org/complement/internal/federation"
 	"github.com/sirupsen/logrus"
 )
 
@@ -29,8 +30,20 @@ func TestMain(m *testing.M) {
 	}
 	// remove any old images/containers/networks in case we died horribly before
 	builder.Cleanup()
+
+	if os.Getenv("COMPLEMENT_CA") == "true" {
+		log.Printf("Running with Complement CA")
+		// make sure CA certs are generated
+		_, _, err = federation.GetOrCreateCaCert()
+		if err != nil {
+			fmt.Printf("Error: %s", err)
+			os.Exit(1)
+		}
+	}
+
 	// we use GMSL which uses logrus by default. We don't want those logs in our test output unless they are Serious.
 	logrus.SetLevel(logrus.ErrorLevel)
+
 	exitCode := m.Run()
 	builder.Cleanup()
 	os.Exit(exitCode)