From e9bd1d8b1f16f1fbaca0d08eba3279bff20fb21b Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Fri, 17 Apr 2020 13:03:33 -0400 Subject: [PATCH] Add new cfssl image and update e2e tests to use it --- images/README.md | 1 + images/cfssl/Makefile | 33 ++++++ images/cfssl/rootfs/Dockerfile | 24 +++++ images/e2e/Dockerfile | 4 + test/e2e-image/Dockerfile | 4 +- test/e2e/run.sh | 4 +- test/e2e/settings/ocsp/ocsp.go | 98 ++++-------------- .../settings/ocsp/{empty.db => template.db} | Bin 8 files changed, 85 insertions(+), 83 deletions(-) create mode 100644 images/cfssl/Makefile create mode 100644 images/cfssl/rootfs/Dockerfile rename test/e2e/settings/ocsp/{empty.db => template.db} (100%) diff --git a/images/README.md b/images/README.md index bf4ed920e..3016fcb3b 100644 --- a/images/README.md +++ b/images/README.md @@ -9,5 +9,6 @@ fastcgi-helloserver | FastCGI application for e2e tests grpc-fortune-teller | grpc server application for the nginx-ingress grpc example httpbin | A simple HTTP Request & Response Service for e2e tests nginx | NGINX base image using [alpine linux](https://www.alpinelinux.org) +cfssl | Image to run cfssl commands :bangbang: Only the nginx image is meant to be published. The others are used as examples for some feature of the ingress controller or to run e2e tests. diff --git a/images/cfssl/Makefile b/images/cfssl/Makefile new file mode 100644 index 000000000..39acad214 --- /dev/null +++ b/images/cfssl/Makefile @@ -0,0 +1,33 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Docker image for e2e testing. + +# Use the 0.0 tag for testing, it shouldn't clobber any release builds +TAG ?= 0.0 + +REGISTRY ?= ingress-controller +DOCKER ?= docker + +IMGNAME = cfssl +IMAGE = $(REGISTRY)/$(IMGNAME) + +container: + $(DOCKER) buildx build \ + --load \ + --platform linux/amd64 \ + -t $(IMAGE):$(TAG) rootfs + +clean: + $(DOCKER) rmi -f $(IMAGE):$(TAG) || true diff --git a/images/cfssl/rootfs/Dockerfile b/images/cfssl/rootfs/Dockerfile new file mode 100644 index 000000000..62db9ec37 --- /dev/null +++ b/images/cfssl/rootfs/Dockerfile @@ -0,0 +1,24 @@ +# Copyright 2019 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:3.11 + +RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +RUN apk add --no-cache \ + bash \ + cfssl@testing + +EXPOSE 8888 + +CMD ["cfssl"] diff --git a/images/e2e/Dockerfile b/images/e2e/Dockerfile index 6b88914a1..e62160a60 100644 --- a/images/e2e/Dockerfile +++ b/images/e2e/Dockerfile @@ -138,4 +138,8 @@ RUN wget https://github.com/helm/chart-testing/releases/download/v${CHART_TESTIN RUN curl -sSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash +RUN curl -sSL -o /usr/local/bin/cfssl https://github.com/cloudflare/cfssl/releases/download/v1.4.1/cfssl_1.4.1_linux_amd64 \ + && curl -sSL -o /usr/local/bin/cfssljson https://github.com/cloudflare/cfssl/releases/download/v1.4.1/cfssljson_1.4.1_linux_amd64 \ + && chmod +x /usr/local/bin/cfssl* + WORKDIR $GOPATH diff --git a/test/e2e-image/Dockerfile b/test/e2e-image/Dockerfile index cb19c34d1..52422ff92 100644 --- a/test/e2e-image/Dockerfile +++ b/test/e2e-image/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/kubernetes-ingress-controller/e2e:v04142020-0257068b9 AS BASE +FROM quay.io/kubernetes-ingress-controller/e2e:v04172020-6e8c68d88 AS BASE FROM alpine:3.11 @@ -16,6 +16,8 @@ RUN curl -sSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-hel COPY --from=BASE /go/bin/ginkgo /usr/local/bin/ COPY --from=BASE /usr/local/bin/kubectl /usr/local/bin/ +COPY --from=BASE /usr/local/bin/cfssl /usr/local/bin/ +COPY --from=BASE /usr/local/bin/cfssljson /usr/local/bin/ COPY . / diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 4c16a2339..85dc49f0d 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -86,6 +86,7 @@ make -C ${DIR}/../../ e2e-test-image make -C ${DIR}/../../images/fastcgi-helloserver/ GO111MODULE=\"on\" build container make -C ${DIR}/../../images/echo/ container make -C ${DIR}/../../images/httpbin/ container +make -C ${DIR}/../../images/cfssl/ container " | parallel --joblog /tmp/log {} || EXIT_CODE=$? if [ ${EXIT_CODE} -eq 0 ] || [ ${EXIT_CODE} -eq -1 ]; then @@ -104,7 +105,6 @@ docker tag ${REGISTRY}/nginx-ingress-controller-${ARCH}:${TAG} ${REGISTRY}/nginx # Preload images used in e2e tests docker pull openresty/openresty:1.15.8.2-alpine docker pull moul/grpcbin -docker pull cfssl/cfssl:1.3.2 echo "[dev-env] copying docker images to cluster..." export EXIT_CODE=-1 @@ -116,7 +116,7 @@ kind load docker-image --name="${KIND_CLUSTER_NAME}" openresty/openresty:1.15.8. kind load docker-image --name="${KIND_CLUSTER_NAME}" ${REGISTRY}/httpbin:${TAG} kind load docker-image --name="${KIND_CLUSTER_NAME}" ${REGISTRY}/echo:${TAG} kind load docker-image --name="${KIND_CLUSTER_NAME}" moul/grpcbin -kind load docker-image --name="${KIND_CLUSTER_NAME}" cfssl/cfssl:1.3.2 +kind load docker-image --name="${KIND_CLUSTER_NAME}" ${REGISTRY}/cfssl:${TAG} " | parallel --joblog /tmp/log {} || EXIT_CODE=$? if [ ${EXIT_CODE} -eq 0 ] || [ ${EXIT_CODE} -eq -1 ]; then diff --git a/test/e2e/settings/ocsp/ocsp.go b/test/e2e/settings/ocsp/ocsp.go index 19f0ceca3..df9007ab8 100644 --- a/test/e2e/settings/ocsp/ocsp.go +++ b/test/e2e/settings/ocsp/ocsp.go @@ -22,12 +22,9 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io" "io/ioutil" "net/http" - "os" "os/exec" - "runtime" "strings" "syscall" "time" @@ -119,7 +116,7 @@ var _ = framework.DescribeSetting("OCSP", func() { }) tlsConfig := &tls.Config{ServerName: host, InsecureSkipVerify: true} - resp := f.HTTPTestClientWithTLSConfig(tlsConfig). + f.HTTPTestClientWithTLSConfig(tlsConfig). GET("/"). WithURL(f.GetURL(framework.HTTPS)). WithHeader("Host", host). @@ -127,8 +124,12 @@ var _ = framework.DescribeSetting("OCSP", func() { Status(http.StatusOK). Raw() - // TODO: avoid second request - resp = f.HTTPTestClientWithTLSConfig(tlsConfig). + // give time the lua request to the OCSP + // URL to finish and update the cache + time.Sleep(5 * time.Second) + + // TODO: is possible to avoid second request? + resp := f.HTTPTestClientWithTLSConfig(tlsConfig). GET("/"). WithURL(f.GetURL(framework.HTTPS)). WithHeader("Host", host). @@ -202,28 +203,16 @@ func prepareCertificates(namespace string) error { return fmt.Errorf("creating cfssl_config.json file: %v", err) } - cfssl := "cfssl" - cfssljson := "cfssljson" - if !commandExists(cfssl) { - ginkgo.By("downloading cfssl...") - cfssl, err = downloadBinary(cfssl) - if err != nil { - return fmt.Errorf("downloading cfssl: %v", err) - } - } - - if !commandExists(cfssljson) { - ginkgo.By("downloading cfssljson...") - cfssljson, err = downloadBinary(cfssljson) - if err != nil { - return fmt.Errorf("downloading cfssljson: %v", err) - } + cpCmd := exec.Command("cp", "-rf", "template.db", "empty.db") + err = cpCmd.Run() + if err != nil { + return fmt.Errorf("copying sqlite file: %v", err) } commands := []string{ - fmt.Sprintf("%v gencert -initca ca_csr.json | %v -bare ca", cfssl, cfssljson), - fmt.Sprintf("%v gencert -ca ca.pem -ca-key ca-key.pem -config=cfssl_config.json -profile=intermediate intermediate_ca_csr.json | %v -bare intermediate_ca", cfssl, cfssljson), - fmt.Sprintf("%v gencert -ca intermediate_ca.pem -ca-key intermediate_ca-key.pem -config=cfssl_config.json -profile=ocsp ocsp_csr.json | %v -bare ocsp", cfssl, cfssljson), + "cfssl gencert -initca ca_csr.json | cfssljson -bare ca", + "cfssl gencert -ca ca.pem -ca-key ca-key.pem -config=cfssl_config.json -profile=intermediate intermediate_ca_csr.json | cfssljson -bare intermediate_ca", + "cfssl gencert -ca intermediate_ca.pem -ca-key intermediate_ca-key.pem -config=cfssl_config.json -profile=ocsp ocsp_csr.json | cfssljson -bare ocsp", } for _, command := range commands { @@ -238,7 +227,7 @@ func prepareCertificates(namespace string) error { ctx, canc := context.WithCancel(context.Background()) defer canc() - command := fmt.Sprintf("%v serve -db-config=db-config.json -ca-key=intermediate_ca-key.pem -ca=intermediate_ca.pem -config=cfssl_config.json -responder=ocsp.pem -responder-key=ocsp-key.pem", cfssl) + command := "cfssl serve -db-config=db-config.json -ca-key=intermediate_ca-key.pem -ca=intermediate_ca.pem -config=cfssl_config.json -responder=ocsp.pem -responder-key=ocsp-key.pem" ginkgo.By(fmt.Sprintf("running %v", command)) serve := exec.CommandContext(ctx, "bash", "-c", command) if err := serve.Start(); err != nil { @@ -248,7 +237,7 @@ func prepareCertificates(namespace string) error { time.Sleep(1 * time.Second) - command = fmt.Sprintf("%v gencert -remote=localhost -profile=server leaf_csr.json | %v -bare leaf", cfssl, cfssljson) + command = "cfssl gencert -remote=localhost -profile=server leaf_csr.json | cfssljson -bare leaf" ginkgo.By(fmt.Sprintf("running %v", command)) out, err := exec.Command("bash", "-c", command).Output() if err != nil { @@ -262,7 +251,7 @@ func prepareCertificates(namespace string) error { return err } - command = fmt.Sprintf("%v ocsprefresh -ca intermediate_ca.pem -responder=ocsp.pem -responder-key=ocsp-key.pem -db-config=db-config.json", cfssl) + command = "cfssl ocsprefresh -ca intermediate_ca.pem -responder=ocsp.pem -responder-key=ocsp-key.pem -db-config=db-config.json" ginkgo.By(fmt.Sprintf("running %v", command)) out, err = exec.Command("bash", "-c", command).Output() if err != nil { @@ -288,57 +277,6 @@ func commandExists(name string) bool { return true } -func getBinary(filepath string, url string) error { - out, err := os.Create(filepath) - if err != nil { - return err - } - - defer out.Close() - - resp, err := http.Get(url) - if err != nil { - return err - } - - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected error downloading CFSSL") - } - - _, err = io.Copy(out, resp.Body) - if err != nil { - return err - } - - return nil -} - -func downloadBinary(name string) (string, error) { - f, err := ioutil.TempFile("", "fw") - if err != nil { - return "", err - } - defer f.Close() - - arch := runtime.GOARCH - goos := runtime.GOOS - cfsslURL := "https://github.com/cloudflare/cfssl/releases/download/v1.4.1/" + name + "_1.4.1_" + goos + "_" + arch - - err = getBinary(f.Name(), cfsslURL) - if err != nil { - return "", err - } - - err = os.Chmod(f.Name(), 0755) - if err != nil { - return "", err - } - - return f.Name(), nil -} - func ocspserveDeployment(namespace string) (*appsv1.Deployment, *corev1.Service) { name := "ocspserve" return &appsv1.Deployment{ @@ -364,7 +302,7 @@ func ocspserveDeployment(namespace string) (*appsv1.Deployment, *corev1.Service) Containers: []corev1.Container{ { Name: name, - Image: "cfssl/cfssl:1.3.2", + Image: "ingress-controller/cfssl:1.0.0-dev", Command: []string{ "/bin/bash", "-c", diff --git a/test/e2e/settings/ocsp/empty.db b/test/e2e/settings/ocsp/template.db similarity index 100% rename from test/e2e/settings/ocsp/empty.db rename to test/e2e/settings/ocsp/template.db