From c39de84a951d1e285c44dc0d721ea9728f84006d Mon Sep 17 00:00:00 2001 From: Ricardo Katz Date: Sun, 28 Jul 2024 19:19:35 -0300 Subject: [PATCH] Add brotli module parser (#11690) --- .../controller/template/crossplane/config.go | 4 +- .../template/crossplane/crossplane_test.go | 66 ++++++++++++++-- .../crossplane/extramodules/README.md | 10 +++ .../crossplane/extramodules/analyze.go | 78 +++++++++++++++++++ .../crossplane/extramodules/brotli.go | 55 +++++++++++++ 5 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 internal/ingress/controller/template/crossplane/extramodules/README.md create mode 100644 internal/ingress/controller/template/crossplane/extramodules/analyze.go create mode 100644 internal/ingress/controller/template/crossplane/extramodules/brotli.go diff --git a/internal/ingress/controller/template/crossplane/config.go b/internal/ingress/controller/template/crossplane/config.go index 369ce2fb3..b754f55dc 100644 --- a/internal/ingress/controller/template/crossplane/config.go +++ b/internal/ingress/controller/template/crossplane/config.go @@ -17,6 +17,8 @@ limitations under the License. package crossplane import ( + "strings" + ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" ) @@ -32,7 +34,7 @@ func (c *Template) buildConfig() { }, } if c.tplConfig.Cfg.WorkerCPUAffinity != "" { - config.Parsed = append(config.Parsed, buildDirective("worker_cpu_affinity", c.tplConfig.Cfg.WorkerCPUAffinity)) + config.Parsed = append(config.Parsed, buildDirective("worker_cpu_affinity", strings.Split(c.tplConfig.Cfg.WorkerCPUAffinity, " "))) } if c.tplConfig.Cfg.EnableBrotli { diff --git a/internal/ingress/controller/template/crossplane/crossplane_test.go b/internal/ingress/controller/template/crossplane/crossplane_test.go index 71765d677..24b98f132 100644 --- a/internal/ingress/controller/template/crossplane/crossplane_test.go +++ b/internal/ingress/controller/template/crossplane/crossplane_test.go @@ -17,6 +17,7 @@ limitations under the License. package crossplane_test import ( + "net" "os" "testing" @@ -25,6 +26,7 @@ import ( "k8s.io/ingress-nginx/internal/ingress/controller/config" "k8s.io/ingress-nginx/internal/ingress/controller/template/crossplane" + "k8s.io/ingress-nginx/internal/ingress/controller/template/crossplane/extramodules" "k8s.io/ingress-nginx/pkg/apis/ingress" ) @@ -36,6 +38,8 @@ types { } ` +var resolvers = []net.IP{net.ParseIP("::1"), net.ParseIP("192.168.20.10")} + // TestTemplate should be a roundtrip test. // We should initialize the scenarios based on the template configuration // Then Parse and write a crossplane configuration, and roundtrip/parse back to check @@ -46,8 +50,12 @@ func TestCrossplaneTemplate(t *testing.T) { options := ngx_crossplane.ParseOptions{ ErrorOnUnknownDirectives: true, StopParsingOnError: true, - IgnoreDirectives: []string{"more_clear_headers"}, - DirectiveSources: []ngx_crossplane.MatchFunc{ngx_crossplane.DefaultDirectivesMatchFunc, ngx_crossplane.LuaDirectivesMatchFn}, + IgnoreDirectives: []string{"more_clear_headers", "more_set_headers"}, // TODO: Add more_set_headers + DirectiveSources: []ngx_crossplane.MatchFunc{ + ngx_crossplane.DefaultDirectivesMatchFunc, + ngx_crossplane.LuaDirectivesMatchFn, + extramodules.BrotliMatchFn, + }, LexOptions: ngx_crossplane.LexOptions{ Lexers: []ngx_crossplane.RegisterLexer{lua.RegisterLexer()}, }, @@ -63,15 +71,43 @@ func TestCrossplaneTemplate(t *testing.T) { require.NoError(t, err) require.NoError(t, mimeFile.Close()) - tplConfig := &config.TemplateConfig{ - Cfg: config.NewDefault(), - } tpl := crossplane.NewTemplate() t.Run("it should be able to marshall and unmarshall the default configuration", func(t *testing.T) { + tplConfig := &config.TemplateConfig{ + Cfg: config.NewDefault(), + } tplConfig.Cfg.DefaultSSLCertificate = defaultCertificate tplConfig.Cfg.EnableBrotli = true tplConfig.Cfg.HideHeaders = []string{"x-fake-header", "x-another-fake-header"} + tplConfig.Cfg.Resolver = resolvers + tplConfig.Cfg.DisableIpv6DNS = true + tplConfig.Cfg.UseForwardedHeaders = true + tplConfig.Cfg.LogFormatEscapeNone = true + tplConfig.Cfg.DisableAccessLog = true + tplConfig.Cfg.UpstreamKeepaliveConnections = 0 + + tpl.SetMimeFile(mimeFile.Name()) + content, err := tpl.Write(tplConfig) + require.NoError(t, err) + + tmpFile, err := os.CreateTemp("", "") + require.NoError(t, err) + _, err = tmpFile.Write(content) + require.NoError(t, err) + require.NoError(t, tmpFile.Close()) + + _, err = ngx_crossplane.Parse(tmpFile.Name(), &options) + require.NoError(t, err) + }) + + t.Run("it should set the right logging configs", func(t *testing.T) { + tplConfig := &config.TemplateConfig{ + Cfg: config.NewDefault(), + } + tplConfig.Cfg.DefaultSSLCertificate = defaultCertificate + tplConfig.Cfg.DisableAccessLog = false + tplConfig.Cfg.HTTPAccessLogPath = "/lalala.log" tpl.SetMimeFile(mimeFile.Name()) content, err := tpl.Write(tplConfig) @@ -88,9 +124,21 @@ func TestCrossplaneTemplate(t *testing.T) { }) t.Run("it should be able to marshall and unmarshall the specified configuration", func(t *testing.T) { + tplConfig := &config.TemplateConfig{ + Cfg: config.NewDefault(), + } tplConfig.Cfg.DefaultSSLCertificate = defaultCertificate + tplConfig.Cfg.WorkerCPUAffinity = "0001 0010 0100 1000" + tplConfig.Cfg.LuaSharedDicts = map[string]int{ + "configuration_data": 10240, + "certificate_data": 50, + } + tplConfig.Cfg.Resolver = resolvers + tplConfig.Cfg.DisableIpv6DNS = false tplConfig.Cfg.UseProxyProtocol = true + tplConfig.Cfg.ProxyRealIPCIDR = []string{"192.168.0.20", "200.200.200.200"} + tplConfig.Cfg.LogFormatEscapeJSON = true tplConfig.Cfg.GRPCBufferSizeKb = 10 // default 0 @@ -107,6 +155,8 @@ func TestCrossplaneTemplate(t *testing.T) { tplConfig.Cfg.DisableHTTPAccessLog = false tplConfig.Cfg.EnableSyslog = true tplConfig.Cfg.SyslogHost = "localhost" + tplConfig.Cfg.SkipAccessLogURLs = []string{"aaa.a", "bbb.b"} + tplConfig.Cfg.SSLDHParam = "/some/dh.pem" // Example: openssl rand 80 | openssl enc -A -base64 tplConfig.Cfg.SSLSessionTicketKey = "lOj3+7Xe21K9GapKqqPIw/gCQm5S4C2lK8pVne6drEik0QqOQHAw1AaPSMdbAvXx2zZKKPCEG98+g3hzftmrfnePSIvokIIE+hHto3Kj1HQ=" @@ -117,6 +167,11 @@ func TestCrossplaneTemplate(t *testing.T) { tplConfig.Cfg.BlockCIDRs = []string{"192.168.0.0/24", " 200.200.0.0/16 "} // default 0 tplConfig.Cfg.BlockUserAgents = []string{"someuseragent", " another/user-agent "} // default 0 + tplConfig.AddHeaders = map[string]string{ + "someheader": "xpto", + "anotherheader": "blabla", + } + tpl = crossplane.NewTemplate() tpl.SetMimeFile(mimeFile.Name()) content, err := tpl.Write(tplConfig) @@ -130,5 +185,6 @@ func TestCrossplaneTemplate(t *testing.T) { _, err = ngx_crossplane.Parse(tmpFile.Name(), &options) require.NoError(t, err) + // require.Equal(t, " ", string(content)) }) } diff --git a/internal/ingress/controller/template/crossplane/extramodules/README.md b/internal/ingress/controller/template/crossplane/extramodules/README.md new file mode 100644 index 000000000..6bd8a4e6b --- /dev/null +++ b/internal/ingress/controller/template/crossplane/extramodules/README.md @@ -0,0 +1,10 @@ +# Extra modules +This folder contains the extra modules used by ingress-nginx and not yet +supported by nginx-go-crossplane + +The generation of the files is done using go-crossplane generator + +## Brotli +``` +go run ./cmd/generate/ -src-path=ngx_brotli/ -directive-map-name=brotliDirectives -match-func-name=BrotliMatchFn > ../ingress-crossplane/internal/ingress/controller/template/crossplane/extramodules/brotli.go +``` \ No newline at end of file diff --git a/internal/ingress/controller/template/crossplane/extramodules/analyze.go b/internal/ingress/controller/template/crossplane/extramodules/analyze.go new file mode 100644 index 000000000..72efe2474 --- /dev/null +++ b/internal/ingress/controller/template/crossplane/extramodules/analyze.go @@ -0,0 +1,78 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * Copyright (c) F5, Inc. + * + * This source code is licensed under the Apache License, Version 2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +// This file is an extraction from https://github.com/nginxinc/nginx-go-crossplane/blob/main/analyze.go +// +//nolint:unused +package extramodules + +// bit masks for different directive argument styles. +const ( + ngxConfNoArgs = 0x00000001 // 0 args + ngxConfTake1 = 0x00000002 // 1 args + ngxConfTake2 = 0x00000004 // 2 args + ngxConfTake3 = 0x00000008 // 3 args + ngxConfTake4 = 0x00000010 // 4 args + ngxConfTake5 = 0x00000020 // 5 args + ngxConfTake6 = 0x00000040 // 6 args + // ngxConfTake7 = 0x00000080 // 7 args (currently unused). + ngxConfBlock = 0x00000100 // followed by block + ngxConfExpr = 0x00000200 // directive followed by expression in parentheses `()` + ngxConfFlag = 0x00000400 // 'on' or 'off' + ngxConfAny = 0x00000800 // >=0 args + ngxConf1More = 0x00001000 // >=1 args + ngxConf2More = 0x00002000 // >=2 args + + // some helpful argument style aliases. + ngxConfTake12 = ngxConfTake1 | ngxConfTake2 + ngxConfTake13 = ngxConfTake1 | ngxConfTake3 + ngxConfTake23 = ngxConfTake2 | ngxConfTake3 + ngxConfTake34 = ngxConfTake3 | ngxConfTake4 + ngxConfTake123 = ngxConfTake12 | ngxConfTake3 + ngxConfTake1234 = ngxConfTake123 | ngxConfTake4 + + // bit masks for different directive locations. + ngxDirectConf = 0x00010000 // main file (not used) + ngxMgmtMainConf = 0x00020000 // mgmt // unique bitmask that may not match NGINX source + ngxMainConf = 0x00040000 // main context + ngxEventConf = 0x00080000 // events + ngxMailMainConf = 0x00100000 // mail + ngxMailSrvConf = 0x00200000 // mail > server + ngxStreamMainConf = 0x00400000 // stream + ngxStreamSrvConf = 0x00800000 // stream > server + ngxStreamUpsConf = 0x01000000 // stream > upstream + ngxHTTPMainConf = 0x02000000 // http + ngxHTTPSrvConf = 0x04000000 // http > server + ngxHTTPLocConf = 0x08000000 // http > location + ngxHTTPUpsConf = 0x10000000 // http > upstream + ngxHTTPSifConf = 0x20000000 // http > server > if + ngxHTTPLifConf = 0x40000000 // http > location > if + ngxHTTPLmtConf = 0x80000000 // http > location > limit_except +) + +// helpful directive location alias describing "any" context +// doesn't include ngxHTTPSifConf, ngxHTTPLifConf, ngxHTTPLmtConf, or ngxMgmtMainConf. +const ngxAnyConf = ngxMainConf | ngxEventConf | ngxMailMainConf | ngxMailSrvConf | + ngxStreamMainConf | ngxStreamSrvConf | ngxStreamUpsConf | + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxHTTPUpsConf | + ngxHTTPSifConf | ngxHTTPLifConf | ngxHTTPLmtConf diff --git a/internal/ingress/controller/template/crossplane/extramodules/brotli.go b/internal/ingress/controller/template/crossplane/extramodules/brotli.go new file mode 100644 index 000000000..0e2762f98 --- /dev/null +++ b/internal/ingress/controller/template/crossplane/extramodules/brotli.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by generator; DO NOT EDIT. +// All the definitions are extracted from the source code +// Each bit mask describes these behaviors: +// - how many arguments the directive can take +// - whether or not it is a block directive +// - whether this is a flag (takes one argument that's either "on" or "off") +// - which contexts it's allowed to be in + +package extramodules + +var brotliDirectives = map[string][]uint{ + "brotli": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxHTTPLifConf | ngxConfFlag, + }, + "brotli_buffers": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake2, + }, + "brotli_comp_level": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, + "brotli_min_length": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, + "brotli_static": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, + "brotli_types": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConf1More, + }, + "brotli_window": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, +} + + +func BrotliMatchFn(directive string) ([]uint, bool) { + m, ok := brotliDirectives[directive] + return m, ok +} \ No newline at end of file