From 9636e41ffdb9170e0083d17f5025dd3ad47617ed Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 9 Nov 2020 15:03:35 +0000 Subject: [PATCH 1/5] Provide an empty json dict to /createRoom instead of no body (#36) --- tests/rooms_state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rooms_state_test.go b/tests/rooms_state_test.go index a0a33d9b..3907de40 100644 --- a/tests/rooms_state_test.go +++ b/tests/rooms_state_test.go @@ -24,7 +24,7 @@ func TestRoomCreationReportsEventsToMyself(t *testing.T) { userID := "@alice:hs1" alice := deployment.Client(t, "hs1", userID) - roomID := alice.CreateRoom(t, nil) + roomID := alice.CreateRoom(t, struct{}{}) t.Run("parallel", func(t *testing.T) { // sytest: Room creation reports m.room.create to myself From 75d8d4a14db2ca9e7cb0852c57b1b1316efa72ab Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 9 Nov 2020 15:04:10 +0000 Subject: [PATCH 2/5] Up the default version check iterations from 50 to 100 (#34) * Up the default version check iterations from 50 to 100 To help out poor old homeservers like Synapse who take some time to start up the python interpreter. * Remove version check variable from Synapse dockerfile comment --- dockerfiles/Synapse.Dockerfile | 2 +- internal/config/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/Synapse.Dockerfile b/dockerfiles/Synapse.Dockerfile index dcc93c44..022c4138 100644 --- a/dockerfiles/Synapse.Dockerfile +++ b/dockerfiles/Synapse.Dockerfile @@ -9,7 +9,7 @@ # To use it: # # (cd dockerfiles && docker build -t complement-synapse -f Synapse.Dockerfile .) -# COMPLEMENT_VERSION_CHECK_ITERATIONS=100 COMPLEMENT_BASE_IMAGE=complement-synapse go test -v ./tests +# COMPLEMENT_BASE_IMAGE=complement-synapse go test -v ./tests FROM matrixdotorg/synapse:latest diff --git a/internal/config/config.go b/internal/config/config.go index a39a9105..fb19ba90 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -18,7 +18,7 @@ func NewConfigFromEnvVars() *Complement { cfg.BaseImageURI = os.Getenv("COMPLEMENT_BASE_IMAGE") cfg.BaseImageArgs = strings.Split(os.Getenv("COMPLEMENT_BASE_IMAGE_ARGS"), " ") cfg.DebugLoggingEnabled = os.Getenv("COMPLEMENT_DEBUG") == "1" - cfg.VersionCheckIterations = parseEnvWithDefault("COMPLEMENT_VERSION_CHECK_ITERATIONS", 50) + cfg.VersionCheckIterations = parseEnvWithDefault("COMPLEMENT_VERSION_CHECK_ITERATIONS", 100) if cfg.BaseImageURI == "" { panic("COMPLEMENT_BASE_IMAGE must be set") } From ca9bd78d1c4369f6e792e388f75c3f7c2178e55b Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 9 Nov 2020 15:05:21 +0000 Subject: [PATCH 3/5] Federation: return Content-Type header of 'application/json' by default (#35) --- internal/federation/server.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/federation/server.go b/internal/federation/server.go index 898b5405..50c01fa0 100644 --- a/internal/federation/server.go +++ b/internal/federation/server.go @@ -78,6 +78,13 @@ func NewServer(t *testing.T, deployment *docker.Deployment, opts ...func(*Server fetcher, }, } + srv.mux.Use(func(h http.Handler) http.Handler { + // Return a json Content-Type header to all requests by default + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + h.ServeHTTP(w, r) + }) + }) srv.mux.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if srv.UnexpectedRequestsAreErrors { t.Errorf("Server.UnexpectedRequestsAreErrors=true received unexpected request to server: %s %s", req.Method, req.URL.Path) From 9acaf3cd28e2d32024c80a3f04105e2cce7ff1e3 Mon Sep 17 00:00:00 2001 From: Rudi Floren Date: Wed, 11 Nov 2020 11:36:31 +0100 Subject: [PATCH 4/5] Add CA generation and mount it to /ca in containers (#28) * Add CA generation * Update internal/docker/builder.go * Update CA volume mounting logic. * Update CA cert creation process Make sure that the certs are created at test start. * Fix grammar in some comments * Gate Complement PKI using COMPLEMENT_CA. Set COMPLEMENT_CA=true to enable Complement PKI * Update README.md Co-authored-by: Kegsay --- README.md | 26 ++++ dockerfiles/ComplementCIBuildkite.Dockerfile | 4 +- internal/docker/builder.go | 94 ++++++++++++- internal/federation/server.go | 132 ++++++++++++++++++- tests/main_test.go | 13 ++ 5 files changed, 264 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cbf8180e..70f30e67 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,37 @@ $ COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -timeout 30s -run '^( - The homeserver needs to `200 OK` requests to `GET /_matrix/client/versions`. - The homeserver needs to manage its own storage within the image. - The homeserver needs to accept the server name given by the environment variable `SERVER_NAME` at runtime. +- The homeserver can use the CA certificate mounted at /ca to create its own TLS cert (see [Complement PKI](README.md#complement-pki)). #### Why 'Complement'? Because **M***C* = **1** - **M** +#### Complement PKI + +As the Matrix federation protocol expects federation endpoints to be served with valid TLS certs, +Complement will create a self-signed CA cert to use for creating valid TLS certs in homeserver containers. + +To enable it pass `COMPLEMENT_CA=true` to complement or the docker container. +If not used, the homeserver needs to not validate the cert when federating. +To check whether complements runs in PKI mode, `COMPLEMENT_CA` is passed through to the homeserver containers. + +The public key to add to the trusted cert store (e.g., /etc/ca-certificates) is mounted at: `/ca/ca.crt` +The private key to sign the created TLS cert is mounted at: `/ca/ca.key` + +For example, to sign your certificate for the homeserver, run at each container start (Ubuntu): +``` +openssl genrsa -out $SERVER_NAME.key 2048 +openssl req -new -sha256 -key $SERVER_NAME.key -subj "/C=US/ST=CA/O=MyOrg, Inc./CN=$SERVER_NAME" -out $SERVER_NAME.csr +openssl x509 -req -in $SERVER_NAME.csr -CA /ca/ca.crt -CAkey /ca/ca.key -CAcreateserial -out $SERVER_NAME.crt -days 1 -sha256 +``` + +To add the CA cert to your trust store (Ubuntu): +``` +cp /root/ca.crt /usr/local/share/ca-certificates/complement.crt +update-ca-certificates +``` + #### Sytest parity ``` diff --git a/dockerfiles/ComplementCIBuildkite.Dockerfile b/dockerfiles/ComplementCIBuildkite.Dockerfile index c0299463..50901043 100644 --- a/dockerfiles/ComplementCIBuildkite.Dockerfile +++ b/dockerfiles/ComplementCIBuildkite.Dockerfile @@ -7,4 +7,6 @@ FROM golang:1.15-buster RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh ADD https://github.com/matrix-org/complement/archive/master.tar.gz . -RUN tar -xzf master.tar.gz && cd complement-master && go mod download \ No newline at end of file +RUN tar -xzf master.tar.gz && cd complement-master && go mod download + +VOLUME [ "/ca" ] \ No newline at end of file diff --git a/internal/docker/builder.go b/internal/docker/builder.go index 75f299a0..50224c49 100644 --- a/internal/docker/builder.go +++ b/internal/docker/builder.go @@ -15,10 +15,13 @@ package docker import ( "context" + "errors" "fmt" "log" + "io/ioutil" "net/http" "os" + "path" "runtime" "strings" "sync" @@ -27,6 +30,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" client "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" @@ -291,27 +295,115 @@ func (d *Builder) deployBaseImage(blueprintName, hsName, contextStr, networkID s return deployImage(d.Docker, d.BaseImage, d.CSAPIPort, fmt.Sprintf("complement_%s", contextStr), blueprintName, hsName, contextStr, networkID) } +// getCaVolume returns the correct mounts and volumes for providing a CA to homeserver containers. +func getCaVolume(docker *client.Client, ctx context.Context) (map[string]struct{}, []mount.Mount, error) { + var caVolume map[string]struct{} + var caMount []mount.Mount + + if os.Getenv("CI") == "true" { + // When in CI, Complement itself is a container with the CA volume mounted at /ca. + // We need to mount this volume to all homeserver containers to synchronize the CA cert. + // This is needed to establish trust among all containers. + + // Get volume mounted at /ca. First we get the container ID + // /proc/1/cpuset should be /docker/ + cpuset, err := ioutil.ReadFile("/proc/1/cpuset") + if err != nil { + return nil, nil, err + } + if !strings.Contains(string(cpuset), "docker") { + return nil, nil, errors.New("Could not identify container ID using /proc/1/cpuset") + } + cpusetList := strings.Split(strings.TrimSpace(string(cpuset)), "/") + containerId := cpusetList[len(cpusetList)-1] + container, err := docker.ContainerInspect(ctx, containerId) + if err != nil { + return nil, nil, err + } + // Get the volume that matches the destination in our complement container + var volumeName string + for i := range container.Mounts { + if container.Mounts[i].Destination == "/ca" { + volumeName = container.Mounts[i].Name + } + } + if volumeName == "" { + // We did not find a volume. This container might be created without a volume, + // or CI=true is passed but we are not running in a container. + // todo: log that we do not provide a CA volume mount? + return nil, nil, nil + } else { + caVolume = map[string]struct{}{ + "/ca": {}, + } + caMount = []mount.Mount{ + { + Type: mount.TypeVolume, + Source: volumeName, + Target: "/ca", + }, + } + } + } else { + // When not in CI, our CA cert is placed in the current working dir. + // We bind mount this directory to all homeserver containers. + cwd, err := os.Getwd() + if err != nil { + return nil, nil, err + } + caCertificateDirHost := path.Join(cwd, "ca") + if _, err := os.Stat(caCertificateDirHost); os.IsNotExist(err) { + err = os.Mkdir(caCertificateDirHost, 0770) + if err != nil { + return nil, nil, err + } + } + caMount = []mount.Mount{ + { + Type: mount.TypeBind, + Source: path.Join(cwd, "ca"), + Target: "/ca", + }, + } + } + return caVolume, caMount, nil +} + func deployImage(docker *client.Client, imageID string, csPort int, containerName, blueprintName, hsName, contextStr, networkID string) (*HomeserverDeployment, error) { ctx := context.Background() var extraHosts []string + var caVolume map[string]struct{} + var caMount []mount.Mount + var err error + if runtime.GOOS == "linux" { // By default docker for linux does not expose this, so do it now. // When https://github.com/moby/moby/pull/40007 lands in Docker 20, we should // change this to be `host.docker.internal:host-gateway` extraHosts = []string{HostnameRunningComplement + ":172.17.0.1"} } + + if os.Getenv("COMPLEMENT_CA") == "true" { + caVolume, caMount, err = getCaVolume(docker, ctx) + if err != nil { + return nil, err + } + } + body, err := docker.ContainerCreate(ctx, &container.Config{ Image: imageID, - Env: []string{"SERVER_NAME=" + hsName}, + Env: []string{"SERVER_NAME=" + hsName, "COMPLEMENT_CA=" + os.Getenv("COMPLEMENT_CA")}, //Cmd: d.ImageArgs, Labels: map[string]string{ complementLabel: contextStr, "complement_blueprint": blueprintName, "complement_hs_name": hsName, }, + Volumes: caVolume, }, &container.HostConfig{ PublishAllPorts: true, ExtraHosts: extraHosts, + Mounts: caMount, }, &network.NetworkingConfig{ map[string]*network.EndpointSettings{ hsName: { diff --git a/internal/federation/server.go b/internal/federation/server.go index 50c01fa0..716fab4a 100644 --- a/internal/federation/server.go +++ b/internal/federation/server.go @@ -9,7 +9,9 @@ import ( "crypto/x509" "encoding/json" "encoding/pem" + "errors" "fmt" + "io/ioutil" "math/big" "net/http" "os" @@ -202,8 +204,119 @@ func (s *Server) Listen() (cancel func()) { } } +// Get or create local CA cert. This is used to create the federation TLS cert. +// In addition, it is passed to homeserver containers to create TLS certs +// for the homeservers +// This basically acts as a test only valid PKI. +func GetOrCreateCaCert() (*x509.Certificate, *rsa.PrivateKey, error) { + var tlsCACertPath, tlsCAKeyPath string + if os.Getenv("CI") == "true" { + // When in CI we create the cert dir in the root directory instead. + tlsCACertPath = path.Join("/ca", "ca.crt") + tlsCAKeyPath = path.Join("/ca", "ca.key") + } else { + wd, err := os.Getwd() + if err != nil { + return nil, nil, err + } + tlsCACertPath = path.Join(wd, "ca", "ca.crt") + tlsCAKeyPath = path.Join(wd,"ca", "ca.key") + if _, err := os.Stat(path.Join(wd, "ca")); os.IsNotExist(err) { + err = os.Mkdir(path.Join(wd, "ca"), 0770) + if err != nil { + return nil, nil, err + } + } + } + if _, err := os.Stat(tlsCACertPath); err == nil { + if _, err := os.Stat(tlsCAKeyPath); err == nil { + // We already created a CA cert, let's use that. + dat, err := ioutil.ReadFile(tlsCACertPath) + if err != nil { + return nil, nil, err + } + block, _ := pem.Decode([]byte(dat)) + if block == nil || block.Type != "CERTIFICATE" { + return nil, nil, errors.New("ca.crt is not a valid pem encoded x509 cert") + } + caCerts, err := x509.ParseCertificates(block.Bytes) + if err != nil { + return nil, nil, err + } + if len(caCerts) != 1 { + return nil, nil, errors.New("ca.crt contains none or more than one cert") + } + caCert := caCerts[0] + dat, err = ioutil.ReadFile(tlsCAKeyPath) + if err != nil { + return nil, nil, err + } + block, _ = pem.Decode([]byte(dat)) + if block == nil || block.Type != "RSA PRIVATE KEY" { + return nil, nil, errors.New("ca.key is not a valid pem encoded rsa private key") + } + priv, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, nil, err + } + return caCert, priv, nil + } + } + + certificateDuration := time.Hour * 5 + priv, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + notBefore := time.Now() + notAfter := notBefore.Add(certificateDuration) + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, nil, err + } + caCert := x509.Certificate{ + SerialNumber: serialNumber, + NotBefore: notBefore, + NotAfter: notAfter, + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature | x509.KeyUsageCRLSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &caCert, &caCert, &priv.PublicKey, priv) + if err != nil { + return nil, nil, err + } + certOut, err := os.Create(tlsCACertPath) + if err != nil { + return nil, nil, err + } + + defer certOut.Close() // nolint: errcheck + if err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return nil, nil, err + } + + keyOut, err := os.OpenFile(tlsCAKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return nil, nil, err + } + defer keyOut.Close() // nolint: errcheck + err = pem.Encode(keyOut, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(priv), + }) + if err != nil { + return nil, nil, err + } + return &caCert, priv, nil +} + // federationServer creates a federation server with the given handler func federationServer(name string, h http.Handler) (*http.Server, string, string, error) { + var derBytes []byte srv := &http.Server{ Addr: ":8448", Handler: h, @@ -231,10 +344,23 @@ func federationServer(name string, h http.Handler) (*http.Server, string, string ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true, } - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) - if err != nil { - return nil, "", "", err + if os.Getenv("COMPLEMENT_CA") == "true" { + // Gate COMPLEMENT_CA + ca, caPrivKey, err := GetOrCreateCaCert() + if err != nil { + return nil, "", "", err + } + derBytes, err = x509.CreateCertificate(rand.Reader, &template, ca, &priv.PublicKey, caPrivKey) + if err != nil { + return nil, "", "", err + } + } else { + derBytes, err = x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, "", "", err + } } + certOut, err := os.Create(tlsCertPath) if err != nil { return nil, "", "", err diff --git a/tests/main_test.go b/tests/main_test.go index 887eb4bb..f2c8c992 100644 --- a/tests/main_test.go +++ b/tests/main_test.go @@ -12,6 +12,7 @@ import ( "github.com/matrix-org/complement/internal/b" "github.com/matrix-org/complement/internal/config" "github.com/matrix-org/complement/internal/docker" + "github.com/matrix-org/complement/internal/federation" "github.com/sirupsen/logrus" ) @@ -29,8 +30,20 @@ func TestMain(m *testing.M) { } // remove any old images/containers/networks in case we died horribly before builder.Cleanup() + + if os.Getenv("COMPLEMENT_CA") == "true" { + log.Printf("Running with Complement CA") + // make sure CA certs are generated + _, _, err = federation.GetOrCreateCaCert() + if err != nil { + fmt.Printf("Error: %s", err) + os.Exit(1) + } + } + // we use GMSL which uses logrus by default. We don't want those logs in our test output unless they are Serious. logrus.SetLevel(logrus.ErrorLevel) + exitCode := m.Run() builder.Cleanup() os.Exit(exitCode) From 9366ba5de734f7093efd09e7e0659b6c868171b3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 11 Nov 2020 15:00:34 +0000 Subject: [PATCH 5/5] Add request query parameter map to instruction struct (#37) This allows passing query parameters to a request while executing an instruction. --- internal/instruction/runner.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/instruction/runner.go b/internal/instruction/runner.go index 4b841086..81de3cb1 100644 --- a/internal/instruction/runner.go +++ b/internal/instruction/runner.go @@ -109,11 +109,14 @@ func (r *Runner) next(instrs []instruction, hsURL string, i int) (*http.Request, r.log("Stopping. Failed to form NewRequest for instruction: %s -- %+v \n", err, instr) return nil, nil, 0 } + q := req.URL.Query() if instr.accessToken != "" { - q := req.URL.Query() q.Set("access_token", r.lookup[instr.accessToken]) - req.URL.RawQuery = q.Encode() } + for paramName, paramValue := range instr.queryParams { + q.Set(paramName, paramValue) + } + req.URL.RawQuery = q.Encode() return req, &instr, i } @@ -125,6 +128,8 @@ type instruction struct { // The HTTP path, starting with '/', without the base URL. Will have placeholder values of the form $foo which need // to be replaced based on 'substitutions'. path string + // Any HTTP query parameters to use in the request + queryParams map[string]string // The HTTP body which will be JSON.Marshal'd body interface{} // The access_token to use in the request, represented as a key to use in the lookup table e.g "user_@alice:localhost"