Skip to content

tests: TestTLSChallenge improvements #12305

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
porridge merged 4 commits into from
Aug 8, 2024
Merged

tests: TestTLSChallenge improvements #12305

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
83369f7
tests: specify img pull secret in pod spec
porridge Aug 7, 2024
98afe97
Shorter timeouts to fit in 30m
porridge Aug 7, 2024
e76d3be
use the const
porridge Aug 7, 2024
e6b60e6
mustGetEnv
porridge Aug 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions tests/cert_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,7 @@ func TestCustomCert(t *testing.T) {
t.Skip("No test CA pem specified")
}

centralCN := os.Getenv("ROX_TEST_CENTRAL_CN")
require.NotEmpty(t, centralCN)
centralCN := mustGetEnv(t, "ROX_TEST_CENTRAL_CN")

trustPool := x509.NewCertPool()
ok := trustPool.AppendCertsFromPEM([]byte(testCentralCertCAPEM))
Expand Down
3 changes: 1 addition & 2 deletions tests/client_ca_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,7 @@ func TestClientCAAuthWithMultipleVerifiedChains(t *testing.T) {
func TestClientCARequested(t *testing.T) {
t.Parallel()

clientCAFile := os.Getenv("CLIENT_CA_PATH")
require.NotEmpty(t, clientCAFile, "no client CA file path set")
clientCAFile := mustGetEnv(t, "CLIENT_CA_PATH")
pemBytes, err := os.ReadFile(clientCAFile)
require.NoErrorf(t, err, "Could not read client CA file %s", clientCAFile)
caCert, err := helpers.ParseCertificatePEM(pemBytes)
Expand Down
9 changes: 8 additions & 1 deletion tests/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func testContexts(t *testing.T, name string, timeout time.Duration) (testCtx con
overallCancel func()
testCancel func()
)
cleanupTimeout := 15 * time.Minute
cleanupTimeout := 10 * time.Minute
t.Logf("Running %s with a timeout of %s plus %s for cleanup", name, timeout, cleanupTimeout)
overallTimeout := timeout + cleanupTimeout
overallErr := fmt.Errorf("overall %s test+cleanup timeout of %s reached", name, overallTimeout)
Expand All @@ -83,6 +83,13 @@ func testContexts(t *testing.T, name string, timeout time.Duration) (testCtx con
return
}

// mustGetEnv calls os.GetEnv and fails the test if result is empty.
func mustGetEnv(t *testing.T, varName string) string {
val := os.Getenv(varName)
require.NotEmptyf(t, val, "Environment variable %q must be set.", varName)
return val
}

func retrieveDeployment(service v1.DeploymentServiceClient, deploymentID string) (*storage.Deployment, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
Expand Down
10 changes: 5 additions & 5 deletions tests/endpoints_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,27 +353,27 @@ func TestEndpoints(t *testing.T) {
if os.Getenv("ORCHESTRATOR_FLAVOR") == "openshift" {
t.Skip("Skipping endpoints test on OCP: TODO(ROX-24688)")
}
userCert, err := tls.LoadX509KeyPair(os.Getenv("CLIENT_CERT_PATH"), os.Getenv("CLIENT_KEY_PATH"))
userCert, err := tls.LoadX509KeyPair(mustGetEnv(t, "CLIENT_CERT_PATH"), mustGetEnv(t, "CLIENT_KEY_PATH"))
require.NoError(t, err, "failed to load user certificate")

serviceCert, err := tls.LoadX509KeyPair(os.Getenv("SERVICE_CERT_FILE"), os.Getenv("SERVICE_KEY_FILE"))
serviceCert, err := tls.LoadX509KeyPair(mustGetEnv(t, "SERVICE_CERT_FILE"), mustGetEnv(t, "SERVICE_KEY_FILE"))
require.NoError(t, err, "failed to load service certificate")

trustPool := x509.NewCertPool()
serviceCAPEMBytes, err := os.ReadFile(os.Getenv("SERVICE_CA_FILE"))
serviceCAPEMBytes, err := os.ReadFile(mustGetEnv(t, "SERVICE_CA_FILE"))
require.NoError(t, err, "failed to load service CA file")
serviceCACert, err := helpers.ParseCertificatePEM(serviceCAPEMBytes)
require.NoError(t, err, "failed to parse service CA cert")
trustPool.AddCert(serviceCACert)

defaultCAPEMBytes, err := os.ReadFile(os.Getenv("DEFAULT_CA_FILE"))
defaultCAPEMBytes, err := os.ReadFile(mustGetEnv(t, "DEFAULT_CA_FILE"))
require.NoError(t, err, "failed to load default CA file")
defaultCACert, err := helpers.ParseCertificatePEM(defaultCAPEMBytes)
require.NoError(t, err, "failed to parse default CA cert")
trustPool.AddCert(defaultCACert)

defaultCertDNSName := os.Getenv("ROX_TEST_CENTRAL_CN")
require.NotEmpty(t, defaultCertDNSName, "missing default certificate DNS name")
require.NotEmpty(t, defaultCertDNSName, "missing default certificate DNS name: $ROX_TEST_CENTRAL_CN")

testCtx := &endpointsTestContext{
allServerNames: []string{defaultCertDNSName, "central.stackrox"},
Expand Down
54 changes: 26 additions & 28 deletions tests/tls_challenge_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
_ "embed"
"encoding/json"
"fmt"
"os"
"regexp"
"testing"
"time"
Expand All @@ -20,15 +19,15 @@ import (
v1 "k8s.io/api/core/v1"
apiErrors "k8s.io/apimachinery/pkg/api/errors"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)

const (
s = namespaces.StackRox // for brevity
proxyNs = "qa-tls-challenge" // Must match the additionalCA X509v3 Subject Alternative Name
sensorDeployment = "sensor"
sensorContainer = "sensor"
centralEndpointVar = "ROX_CENTRAL_ENDPOINT"
s = namespaces.StackRox // for brevity
proxyNs = "qa-tls-challenge" // Must match the additionalCA X509v3 Subject Alternative Name
proxyImagePullSecretName = "quay"
sensorDeployment = "sensor"
sensorContainer = "sensor"
centralEndpointVar = "ROX_CENTRAL_ENDPOINT"
)

//go:embed "bad-ca/root.crt"
Expand All @@ -54,7 +53,7 @@ type TLSChallengeSuite struct {

func (ts *TLSChallengeSuite) SetupSuite() {
ts.KubernetesSuite.SetupSuite()
ts.ctx, ts.cleanupCtx, ts.cancel = testContexts(ts.T(), "TestTLSChallenge", 20*time.Minute)
ts.ctx, ts.cleanupCtx, ts.cancel = testContexts(ts.T(), "TestTLSChallenge", 15*time.Minute)

// Check sanity before test.
waitUntilCentralSensorConnectionIs(ts.T(), ts.ctx, storage.ClusterHealthStatus_HEALTHY)
Expand All @@ -63,7 +62,7 @@ func (ts *TLSChallengeSuite) SetupSuite() {
ts.originalCentralEndpoint = ts.getDeploymentEnvVal(ts.ctx, s, sensorDeployment, sensorContainer, centralEndpointVar)
ts.logf("Original value is %q. (Will restore this value on cleanup.)", ts.originalCentralEndpoint)

ts.setupProxy(proxyNs, ts.originalCentralEndpoint)
ts.setupProxy(ts.originalCentralEndpoint)
}

func (ts *TLSChallengeSuite) TearDownSuite() {
Expand Down Expand Up @@ -95,47 +94,43 @@ func (ts *TLSChallengeSuite) TestTLSChallenge() {
waitUntilCentralSensorConnectionIs(ts.T(), ts.ctx, storage.ClusterHealthStatus_HEALTHY)
}

func (ts *TLSChallengeSuite) setupProxy(proxyNs string, centralEndpoint string) {
func (ts *TLSChallengeSuite) setupProxy(centralEndpoint string) {
name := "nginx-loadbalancer"
nginxLabels := map[string]string{"app": "nginx"}
nginxTLSSecretName := "nginx-tls-conf" //nolint:gosec // G101
nginxConfigName := "nginx-proxy-conf"
ts.logf("Setting up nginx proxy in namespace %q...", proxyNs)
ts.createProxyNamespace(proxyNs)
ts.installImagePullSecret(proxyNs)
ts.createProxyTLSSecret(proxyNs, nginxTLSSecretName)
ts.createProxyConfigMap(proxyNs, centralEndpoint, nginxConfigName)
ts.createProxyNamespace()
ts.installImagePullSecret()
ts.createProxyTLSSecret(nginxTLSSecretName)
ts.createProxyConfigMap(centralEndpoint, nginxConfigName)
ts.createService(ts.ctx, proxyNs, name, nginxLabels, map[int32]int32{443: 8443})
ts.createProxyDeployment(proxyNs, name, nginxLabels, nginxConfigName, nginxTLSSecretName)
ts.createProxyDeployment(name, nginxLabels, nginxConfigName, nginxTLSSecretName)
ts.logf("Nginx proxy is now set up in namespace %q.", proxyNs)
}

func (ts *TLSChallengeSuite) createProxyNamespace(proxyNs string) {
func (ts *TLSChallengeSuite) createProxyNamespace() {
_, err := ts.k8s.CoreV1().Namespaces().Create(ts.ctx, &v1.Namespace{ObjectMeta: metaV1.ObjectMeta{Name: proxyNs}}, metaV1.CreateOptions{})
if apiErrors.IsAlreadyExists(err) {
return
}
ts.Require().NoError(err, "cannot create proxy namespace %q", proxyNs)
}

func (ts *TLSChallengeSuite) installImagePullSecret(proxyNs string) {
func (ts *TLSChallengeSuite) installImagePullSecret() {
configBytes, err := json.Marshal(config.DockerConfigJSON{
Auths: map[string]config.DockerConfigEntry{
"https://quay.io": {
Username: os.Getenv("REGISTRY_USERNAME"),
Password: os.Getenv("REGISTRY_PASSWORD"),
Username: mustGetEnv(ts.T(), "REGISTRY_USERNAME"),
Password: mustGetEnv(ts.T(), "REGISTRY_PASSWORD"),
},
},
})
secretName := "quay"
ts.Require().NoError(err, "cannot serialize docker config for image pull secret %q in namespace %q", secretName, proxyNs)
ts.ensureSecretExists(ts.ctx, proxyNs, secretName, v1.SecretTypeDockerConfigJson, map[string][]byte{v1.DockerConfigJsonKey: configBytes})
patch := []byte(fmt.Sprintf(`{"imagePullSecrets":[{"name":%q}]}`, secretName))
_, err = ts.k8s.CoreV1().ServiceAccounts(proxyNs).Patch(ts.ctx, "default", types.StrategicMergePatchType, patch, metaV1.PatchOptions{})
ts.Require().NoError(err, "cannot patch service account %q in namespace %q", "default", proxyNs)
ts.Require().NoError(err, "cannot serialize docker config for image pull secret %q in namespace %q", proxyImagePullSecretName, proxyNs)
ts.ensureSecretExists(ts.ctx, proxyNs, proxyImagePullSecretName, v1.SecretTypeDockerConfigJson, map[string][]byte{v1.DockerConfigJsonKey: configBytes})
}

func (ts *TLSChallengeSuite) createProxyTLSSecret(proxyNs string, nginxTLSSecretName string) {
func (ts *TLSChallengeSuite) createProxyTLSSecret(nginxTLSSecretName string) {
var certChain []byte
certChain = append(certChain, leafCert...)
certChain = append(certChain, additionalCA...)
Expand All @@ -145,7 +140,7 @@ func (ts *TLSChallengeSuite) createProxyTLSSecret(proxyNs string, nginxTLSSecret
})
}

func (ts *TLSChallengeSuite) createProxyConfigMap(proxyNs string, centralEndpoint string, nginxConfigName string) {
func (ts *TLSChallengeSuite) createProxyConfigMap(centralEndpoint string, nginxConfigName string) {
const nginxConfigTmpl = `
server {
listen 8443 ssl http2;
Expand All @@ -171,7 +166,7 @@ server {
})
}

func (ts *TLSChallengeSuite) createProxyDeployment(proxyNs string, name string, nginxLabels map[string]string, nginxConfigName string, nginxTLSSecretName string) {
func (ts *TLSChallengeSuite) createProxyDeployment(name string, nginxLabels map[string]string, nginxConfigName string, nginxTLSSecretName string) {
d := &appsV1.Deployment{
ObjectMeta: metaV1.ObjectMeta{
Name: name,
Expand All @@ -187,6 +182,9 @@ func (ts *TLSChallengeSuite) createProxyDeployment(proxyNs string, name string,
Labels: nginxLabels,
},
Spec: v1.PodSpec{
ImagePullSecrets: []v1.LocalObjectReference{
{Name: proxyImagePullSecretName},
},
Containers: []v1.Container{
{
Image: "quay.io/rhacs-eng/qa-multi-arch:nginx-1-17-1",
Expand Down