From a124daf51db13b49b383802a613fa0a3a04d2681 Mon Sep 17 00:00:00 2001 From: Daniel Dias Date: Tue, 23 May 2023 16:51:03 -0300 Subject: [PATCH] Improve test output and remove parallelism --- testing/cli-e2etest/Makefile | 2 +- testing/cli-e2etest/environment/manager.go | 11 +++-------- testing/cli-e2etest/helpers/common.go | 9 +++++++++ .../datastore/apply_new_datastore_test.go | 6 +++--- .../testscenarios/datastore/delete_datastore_test.go | 8 ++++---- .../testscenarios/datastore/list_datastore_test.go | 6 +++--- .../environment/apply_new_environment_test.go | 3 ++- 7 files changed, 25 insertions(+), 20 deletions(-) diff --git a/testing/cli-e2etest/Makefile b/testing/cli-e2etest/Makefile index b3e5782c1f..624d222b7b 100644 --- a/testing/cli-e2etest/Makefile +++ b/testing/cli-e2etest/Makefile @@ -5,4 +5,4 @@ help: Makefile ## show list of commands test: # run tests for this application go clean -testcache - go test -timeout 150s ./... + go test -timeout 150s -p 1 ./... diff --git a/testing/cli-e2etest/environment/manager.go b/testing/cli-e2etest/environment/manager.go index e96344b4c4..cdbf685edd 100644 --- a/testing/cli-e2etest/environment/manager.go +++ b/testing/cli-e2etest/environment/manager.go @@ -11,6 +11,7 @@ import ( "github.com/kubeshop/tracetest/cli-e2etest/command" "github.com/kubeshop/tracetest/cli-e2etest/config" + "github.com/kubeshop/tracetest/cli-e2etest/helpers" "github.com/stretchr/testify/require" "golang.org/x/exp/slices" ) @@ -37,8 +38,6 @@ type internalManager struct { } func CreateAndStart(t *testing.T) Manager { - t.Helper() - mutex.Lock() defer mutex.Unlock() @@ -85,8 +84,6 @@ func (m *internalManager) Name() string { } func (m *internalManager) Start(t *testing.T) { - t.Helper() - result, err := command.Exec( "docker", "compose", "--file", m.dockerComposeFilePath, // choose docker compose relative to the chosen environment @@ -94,15 +91,13 @@ func (m *internalManager) Start(t *testing.T) { "up", "--detach") require.NoError(t, err) - require.Equal(t, 0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) // TODO: think in a better way to assure readiness for Tracetest time.Sleep(1000 * time.Millisecond) } func (m *internalManager) Close(t *testing.T) { - t.Helper() - result, err := command.Exec( "docker", "compose", "--file", m.dockerComposeFilePath, // choose docker compose relative to the chosen environment @@ -113,7 +108,7 @@ func (m *internalManager) Close(t *testing.T) { "--stop", // force containers to stop ) require.NoError(t, err) - require.Equal(t, 0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) } func (m *internalManager) GetCLIConfigPath(t *testing.T) string { diff --git a/testing/cli-e2etest/helpers/common.go b/testing/cli-e2etest/helpers/common.go index 06e45750b7..87168c91ce 100644 --- a/testing/cli-e2etest/helpers/common.go +++ b/testing/cli-e2etest/helpers/common.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/goccy/go-yaml" + "github.com/kubeshop/tracetest/cli-e2etest/command" "github.com/stretchr/testify/require" ) @@ -49,3 +50,11 @@ func UnmarshalYAMLSequence[T any](t *testing.T, data string) []T { return result } + +func RequireExitCodeEqual(t *testing.T, result *command.ExecResult, expectedExitCode int) { + require.Equal( + t, expectedExitCode, result.ExitCode, + "command %s finished with wrong exit code. Expected code: %d Obtained code: %d \nStdOut: %s \nStdErr: %s", + result.CommandExecuted, expectedExitCode, result.ExitCode, result.StdOut, result.StdErr, + ) +} diff --git a/testing/cli-e2etest/testscenarios/datastore/apply_new_datastore_test.go b/testing/cli-e2etest/testscenarios/datastore/apply_new_datastore_test.go index 9dd29dbce2..aea4b0b99b 100644 --- a/testing/cli-e2etest/testscenarios/datastore/apply_new_datastore_test.go +++ b/testing/cli-e2etest/testscenarios/datastore/apply_new_datastore_test.go @@ -28,7 +28,7 @@ func TestApplyNewDatastore(t *testing.T) { // Then it should return an empty datastore result := tracetestcli.Exec(t, "get datastore --id current", tracetestcli.WithCLIConfig(cliConfig)) // TODO: we haven't defined a valid output to tell to the user that we are on `no-tracing mode` - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) dataStore := helpers.UnmarshalYAML[types.DataStoreResource](t, result.StdOut) require.Equal("DataStore", dataStore.Type) @@ -39,12 +39,12 @@ func TestApplyNewDatastore(t *testing.T) { dataStorePath := env.GetEnvironmentResourcePath(t, "data-store") result = tracetestcli.Exec(t, fmt.Sprintf("apply datastore --file %s", dataStorePath), tracetestcli.WithCLIConfig(cliConfig)) - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) // When I try to get a datastore again // Then it should return the datastore applied on the last step result = tracetestcli.Exec(t, "get datastore --id current", tracetestcli.WithCLIConfig(cliConfig)) - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) dataStore = helpers.UnmarshalYAML[types.DataStoreResource](t, result.StdOut) require.Equal("DataStore", dataStore.Type) diff --git a/testing/cli-e2etest/testscenarios/datastore/delete_datastore_test.go b/testing/cli-e2etest/testscenarios/datastore/delete_datastore_test.go index 9b2e5879df..0687a77bcb 100644 --- a/testing/cli-e2etest/testscenarios/datastore/delete_datastore_test.go +++ b/testing/cli-e2etest/testscenarios/datastore/delete_datastore_test.go @@ -29,12 +29,12 @@ func TestDeleteDatastore(t *testing.T) { dataStorePath := env.GetEnvironmentResourcePath(t, "data-store") result := tracetestcli.Exec(t, fmt.Sprintf("apply datastore --file %s", dataStorePath), tracetestcli.WithCLIConfig(cliConfig)) - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) // When I try to get a datastore // Then it should return the datastore applied on the last step result = tracetestcli.Exec(t, "get datastore --id current", tracetestcli.WithCLIConfig(cliConfig)) - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) dataStore := helpers.UnmarshalYAML[types.DataStoreResource](t, result.StdOut) require.Equal("DataStore", dataStore.Type) @@ -44,14 +44,14 @@ func TestDeleteDatastore(t *testing.T) { // When I try to delete the datastore // Then it should delete with success result = tracetestcli.Exec(t, "delete datastore --id current", tracetestcli.WithCLIConfig(cliConfig)) - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) require.Contains(result.StdOut, "DataStore removed. Defaulting back to no-tracing mode") // When I try to get a datastore again // Then it should return an empty datastore result = tracetestcli.Exec(t, "get datastore --id current", tracetestcli.WithCLIConfig(cliConfig)) // TODO: we haven't defined a valid output to tell to the user that we are on `no-tracing mode` - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) dataStore = helpers.UnmarshalYAML[types.DataStoreResource](t, result.StdOut) require.Equal("DataStore", dataStore.Type) diff --git a/testing/cli-e2etest/testscenarios/datastore/list_datastore_test.go b/testing/cli-e2etest/testscenarios/datastore/list_datastore_test.go index eb7770178e..d635d8753e 100644 --- a/testing/cli-e2etest/testscenarios/datastore/list_datastore_test.go +++ b/testing/cli-e2etest/testscenarios/datastore/list_datastore_test.go @@ -29,12 +29,12 @@ func TestListDatastore(t *testing.T) { dataStorePath := env.GetEnvironmentResourcePath(t, "data-store") result := tracetestcli.Exec(t, fmt.Sprintf("apply datastore --file %s", dataStorePath), tracetestcli.WithCLIConfig(cliConfig)) - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) // When I try to list datastore again on pretty mode // Then it should print a table with 4 lines printed: header, separator, data store item and empty line result = tracetestcli.Exec(t, "list datastore --output pretty", tracetestcli.WithCLIConfig(cliConfig)) - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) require.Contains(result.StdOut, "current") require.Contains(result.StdOut, env.Name()) @@ -44,7 +44,7 @@ func TestListDatastore(t *testing.T) { // When I try to list datastore again on json mode // Then it should print a JSON list with one item result = tracetestcli.Exec(t, "list datastore --output json", tracetestcli.WithCLIConfig(cliConfig)) - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) dataStoresJSON := helpers.UnmarshalJSON[[]types.DataStoreResource](t, result.StdOut) diff --git a/testing/cli-e2etest/testscenarios/environment/apply_new_environment_test.go b/testing/cli-e2etest/testscenarios/environment/apply_new_environment_test.go index 31aaea078c..19fd8002ce 100644 --- a/testing/cli-e2etest/testscenarios/environment/apply_new_environment_test.go +++ b/testing/cli-e2etest/testscenarios/environment/apply_new_environment_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/kubeshop/tracetest/cli-e2etest/environment" + "github.com/kubeshop/tracetest/cli-e2etest/helpers" "github.com/kubeshop/tracetest/cli-e2etest/tracetestcli" "github.com/stretchr/testify/require" ) @@ -24,7 +25,7 @@ func TestApplyNewEnvironment(t *testing.T) { // When I try to get an environment that doesn't exists // Then it should return error message result := tracetestcli.Exec(t, "get environment --id .noenv", tracetestcli.WithCLIConfig(cliConfig)) - require.Equal(0, result.ExitCode) + helpers.RequireExitCodeEqual(t, result, 0) require.Contains(result.StdOut, "Resource environment with ID .noenv not found") // When I try to set up a new environment