local helpers = require "spec.helpers" local ssl_fixtures = require "spec.fixtures.ssl" -- using the full path so that we don't have to modify package.path in -- this context local test_vault = require "spec.fixtures.custom_vaults.kong.vaults.test" local CUSTOM_VAULTS = "./spec/fixtures/custom_vaults" local CUSTOM_PLUGINS = "./spec/fixtures/custom_plugins" local LUA_PATH = CUSTOM_VAULTS .. "/?.lua;" .. CUSTOM_VAULTS .. "/?/init.lua;" .. CUSTOM_PLUGINS .. "/?.lua;" .. CUSTOM_PLUGINS .. "/?/init.lua;;" local DUMMY_HEADER = "Dummy-Plugin" local fmt = string.format --- A vault test harness is a driver for vault backends, which implements --- all the necessary glue for initializing a vault backend and performing --- secret read/write operations. --- --- All functions defined here are called as "methods" (e.g. harness:fn()), so --- it is permitted to keep state on the harness object (self). --- ---@class vault_test_harness --- ---@field name string --- --- this table is passed directly to kong.db.vaults:insert() ---@field config table --- --- create_secret() is called once per test run for a given secret ---@field create_secret fun(self: vault_test_harness, secret: string, value: string, opts?: table) --- --- update_secret() may be called more than once per test run for a given secret ---@field update_secret fun(self: vault_test_harness, secret: string, value: string, opts?: table) --- --- setup() is called before kong is started and before any DB entities --- have been created and is best used for things like validating backend --- credentials and establishing a connection to a backend ---@field setup fun(self: vault_test_harness) --- --- teardown() is exactly what you'd expect ---@field teardown fun(self: vault_test_harness) --- --- fixtures() output is passed directly to `helpers.start_kong()` ---@field fixtures fun(self: vault_test_harness):table|nil --- --- ---@field prefix string # generated by the test suite ---@field host string # generated by the test suite ---@type vault_test_harness[] local VAULTS = { { name = "test", config = { default_value = "DEFAULT", default_value_ttl = 1, }, create_secret = function(self, _, value) -- Currently, create_secret is called _before_ starting Kong. -- -- This means our backend won't be available yet because it is -- piggy-backing on Kong as an HTTP mock fixture. -- -- We can, however, inject a default value into our configuration. self.config.default_value = value end, update_secret = function(_, secret, value, opts) return test_vault.client.put(secret, value, opts) end, fixtures = function() return { http_mock = { test_vault = test_vault.http_mock, } } end, }, } local noop = function(...) end for _, vault in ipairs(VAULTS) do -- fill out some values that we'll use in route/service/plugin config vault.prefix = vault.name .. "-ttl-test" vault.host = vault.name .. ".vault-ttl.test" -- ...and fill out non-required methods vault.setup = vault.setup or noop vault.teardown = vault.teardown or noop vault.fixtures = vault.fixtures or noop end for _, strategy in helpers.each_strategy() do for _, vault in ipairs(VAULTS) do describe("vault ttl and rotation (#" .. strategy .. ") #" .. vault.name, function() local client local secret = "my-secret" local function http_get(path) path = path or "/" local res = client:get(path, { headers = { host = assert(vault.host), }, }) assert.response(res).has.status(200) return res end lazy_setup(function() helpers.setenv("KONG_LUA_PATH_OVERRIDE", LUA_PATH) helpers.setenv("KONG_VAULT_ROTATION_INTERVAL", "1") vault:setup() vault:create_secret(secret, "init") local bp = helpers.get_db_utils(strategy, { "vaults", "routes", "services", "plugins" }, { "dummy" }, { vault.name }) assert(bp.vaults:insert({ name = vault.name, prefix = vault.prefix, config = vault.config, })) local route = assert(bp.routes:insert({ name = vault.host, hosts = { vault.host }, paths = { "/" }, service = assert(bp.services:insert()), })) -- used by the plugin config test case assert(bp.plugins:insert({ name = "dummy", config = { resp_header_value = fmt("{vault://%s/%s?ttl=%s}", vault.prefix, secret, 10), }, route = { id = route.id }, })) assert(helpers.start_kong({ database = strategy, nginx_conf = "spec/fixtures/custom_nginx.template", vaults = vault.name, plugins = "dummy", log_level = "info", }, nil, nil, vault:fixtures() )) client = helpers.proxy_client() end) lazy_teardown(function() if client then client:close() end helpers.stop_kong() vault:teardown() helpers.unsetenv("KONG_LUA_PATH_OVERRIDE") end) it("updates plugin config references (backend: #" .. vault.name .. ")", function() local function check_plugin_secret(expect, ttl, leeway) leeway = leeway or 0.25 -- 25% local timeout = ttl + (ttl * leeway) assert .with_timeout(timeout) .with_step(0.5) .eventually(function() local res = http_get("/") local value = assert.response(res).has.header(DUMMY_HEADER) if value == expect then return true end return nil, { expected = expect, got = value } end) .is_truthy("expected plugin secret to be updated to '" .. expect .. "' " .. "' within " .. tostring(timeout) .. "seconds") end vault:update_secret(secret, "old", { ttl = 5 }) check_plugin_secret("old", 5) vault:update_secret(secret, "new", { ttl = 5 }) check_plugin_secret("new", 5) end) end) describe("vault rotation #without ttl (#" .. strategy .. ") #" .. vault.name, function() local client local secret = "my-secret" local function http_get(path) path = path or "/" local res = client:get(path, { headers = { host = assert(vault.host), }, }) assert.response(res).has.status(200) return res end lazy_setup(function() helpers.setenv("KONG_LUA_PATH_OVERRIDE", LUA_PATH) helpers.setenv("KONG_VAULT_ROTATION_INTERVAL", "1") vault:setup() local bp = helpers.get_db_utils(strategy, { "vaults", "routes", "services", "plugins" }, { "dummy" }, { vault.name }) -- override a default config without default ttl assert(bp.vaults:insert({ name = vault.name, prefix = vault.prefix, config = { default_value = "init", }, })) local route = assert(bp.routes:insert({ name = vault.host, hosts = { vault.host }, paths = { "/" }, service = assert(bp.services:insert()), })) -- used by the plugin config test case assert(bp.plugins:insert({ name = "dummy", config = { resp_header_value = fmt("{vault://%s/%s}", vault.prefix, secret), }, route = { id = route.id }, })) assert(helpers.start_kong({ database = strategy, nginx_conf = "spec/fixtures/custom_nginx.template", vaults = vault.name, plugins = "dummy", log_level = "info", }, nil, nil, vault:fixtures() )) client = helpers.proxy_client() end) lazy_teardown(function() if client then client:close() end helpers.stop_kong() vault:teardown() helpers.unsetenv("KONG_LUA_PATH_OVERRIDE") end) it("update secret value should not refresh cached vault reference(backend: #" .. vault.name .. ")", function() local function check_plugin_secret(expect, ttl, leeway) leeway = leeway or 0.25 -- 25% local timeout = ttl + (ttl * leeway) -- The secret value is supposed to be not refreshed -- after several rotations assert.has_error(function() assert .with_timeout(timeout) .with_step(0.5) .eventually(function() local res = http_get("/") local value = assert.response(res).has.header(DUMMY_HEADER) if value == expect then return true end return false end) .is_falsy("expected plugin secret not to be updated to '" .. expect .. "' " .. "' within " .. tostring(timeout) .. "seconds") end) end vault:update_secret(secret, "old") check_plugin_secret("init", 5) vault:update_secret(secret, "new") check_plugin_secret("init", 5) end) end) describe("#hybrid mode dp vault ttl and rotation (#" .. strategy .. ") #" .. vault.name, function() local client local admin_client local secret = "my-secret" local certificate local tls_fixtures = { http_mock = { upstream_tls = [[ server { server_name example.com; listen 16799 ssl; ssl_certificate ../spec/fixtures/mtls_certs/example.com.crt; ssl_certificate_key ../spec/fixtures/mtls_certs/example.com.key; ssl_client_certificate ../spec/fixtures/mtls_certs/ca.crt; ssl_verify_client on; ssl_verify_depth 3; ssl_session_tickets off; ssl_session_cache off; keepalive_requests 0; location = / { echo 'it works'; } } ]] }, } tls_fixtures.dns_mock = helpers.dns_mock.new({mocks_only = true}) tls_fixtures.dns_mock:A { name = "example.com", address = "127.0.0.1", } local vault_fixtures = vault:fixtures() vault_fixtures.dns_mock = tls_fixtures.dns_mock describe("rotation", function() lazy_setup(function() helpers.setenv("KONG_LUA_PATH_OVERRIDE", LUA_PATH) helpers.setenv("KONG_VAULT_ROTATION_INTERVAL", "1") vault:setup() vault:create_secret(secret, ssl_fixtures.key_alt) local bp = helpers.get_db_utils(strategy, { "vaults", "routes", "services", "certificates", "ca_certificates" }, {}, { vault.name }) assert(bp.vaults:insert({ name = vault.name, prefix = vault.prefix, config = vault.config, })) -- Prepare TLS upstream service -- cert_alt & key_alt pair is not a correct client certificate -- and it will fail the client TLS verification on server side -- -- On the other hand, cert_client & key_client pair is a correct -- client certificate certificate = assert(bp.certificates:insert({ key = ssl_fixtures.key_alt, cert = ssl_fixtures.cert_alt, })) local service_tls = assert(bp.services:insert({ name = "tls-service", url = "https://example.com:16799", client_certificate = certificate, })) assert(bp.routes:insert({ name = "tls-route", hosts = { "example.com" }, paths = { "/tls", }, service = { id = service_tls.id }, })) assert(helpers.start_kong({ role = "control_plane", cluster_cert = "spec/fixtures/kong_clustering.crt", cluster_cert_key = "spec/fixtures/kong_clustering.key", database = strategy, prefix = "vault_ttl_test_cp", cluster_listen = "127.0.0.1:9005", admin_listen = "127.0.0.1:9001", nginx_conf = "spec/fixtures/custom_nginx.template", vaults = vault.name, plugins = "dummy", log_level = "debug", }, nil, nil, tls_fixtures )) assert(helpers.start_kong({ role = "data_plane", database = "off", prefix = "vault_ttl_test_dp", vaults = vault.name, plugins = "dummy", log_level = "debug", nginx_conf = "spec/fixtures/custom_nginx.template", cluster_cert = "spec/fixtures/kong_clustering.crt", cluster_cert_key = "spec/fixtures/kong_clustering.key", cluster_control_plane = "127.0.0.1:9005", proxy_listen = "127.0.0.1:9002", nginx_worker_processes = 1, }, nil, nil, vault_fixtures )) admin_client = helpers.admin_client(nil, 9001) client = helpers.proxy_client(nil, 9002) end) lazy_teardown(function() if client then client:close() end if admin_client then admin_client:close() end helpers.stop_kong("vault_ttl_test_cp") helpers.stop_kong("vault_ttl_test_dp") vault:teardown() helpers.unsetenv("KONG_LUA_PATH_OVERRIDE") end) it("updates plugin config references (backend: #" .. vault.name .. ")", function() helpers.wait_for_all_config_update({ forced_admin_port = 9001, forced_proxy_port = 9002, }) -- Wrong cert-key pair is being used in the pre-configured cert object local res = client:get("/tls", { headers = { host = "example.com", }, timeout = 2, }) local body = assert.res_status(400, res) assert.matches("The SSL certificate error", body) -- Switch to vault referenced key field local res = assert(admin_client:patch("/certificates/"..certificate.id, { body = { key = fmt("{vault://%s/%s?ttl=%s}", vault.prefix, secret, 2), cert = ssl_fixtures.cert_client, }, headers = { ["Content-Type"] = "application/json", }, })) assert.res_status(200, res) helpers.wait_for_all_config_update({ forced_admin_port = 9001, forced_proxy_port = 9002, }) -- Assume wrong cert-key pair still being used local res = client:get("/tls", { headers = { host = "example.com", }, timeout = 2, }) local body = assert.res_status(400, res) assert.matches("No required SSL certificate was sent", body) -- Update secret value and let cert be correct vault:update_secret(secret, ssl_fixtures.key_client, { ttl = 2 }) assert.with_timeout(7) .with_step(0.5) .ignore_exceptions(true) .eventually(function() local res = client:get("/tls", { headers = { host = "example.com", }, timeout = 2, }) local body = assert.res_status(200, res) assert.matches("it works", body) return true end).is_truthy("Expected certificate being refreshed") end) end) describe("rotation", function() lazy_setup(function() helpers.setenv("KONG_LUA_PATH_OVERRIDE", LUA_PATH) helpers.setenv("KONG_VAULT_ROTATION_INTERVAL", "1") vault:setup() vault:create_secret(secret, ssl_fixtures.key_alt) local bp = helpers.get_db_utils(strategy, { "vaults", "routes", "services", "certificates", "ca_certificates" }, {}, { vault.name }) assert(bp.vaults:insert({ name = vault.name, prefix = vault.prefix, config = vault.config, })) -- Prepare TLS upstream service -- cert_alt & key_alt pair is not a correct client certificate -- and it will fail the client TLS verification on server side -- -- On the other hand, cert_client & key_client pair is a correct -- client certificate certificate = assert(bp.certificates:insert({ key = ssl_fixtures.key_alt, cert = ssl_fixtures.cert_alt, })) local service_tls = assert(bp.services:insert({ name = "tls-service", url = "https://example.com:16799", client_certificate = certificate, })) assert(bp.routes:insert({ name = "tls-route", hosts = { "example.com" }, paths = { "/tls", }, service = { id = service_tls.id }, })) assert(helpers.start_kong({ role = "control_plane", cluster_cert = "spec/fixtures/kong_clustering.crt", cluster_cert_key = "spec/fixtures/kong_clustering.key", database = strategy, prefix = "vault_ttl_test_cp", cluster_listen = "127.0.0.1:9005", admin_listen = "127.0.0.1:9001", nginx_conf = "spec/fixtures/custom_nginx.template", vaults = vault.name, plugins = "dummy", log_level = "debug", }, nil, nil, tls_fixtures )) assert(helpers.start_kong({ role = "data_plane", database = "off", prefix = "vault_ttl_test_dp", vaults = vault.name, plugins = "dummy", log_level = "debug", nginx_conf = "spec/fixtures/custom_nginx.template", cluster_cert = "spec/fixtures/kong_clustering.crt", cluster_cert_key = "spec/fixtures/kong_clustering.key", cluster_control_plane = "127.0.0.1:9005", proxy_listen = "127.0.0.1:9002", nginx_worker_processes = 1, }, nil, nil, vault_fixtures )) admin_client = helpers.admin_client(nil, 9001) client = helpers.proxy_client(nil, 9002) end) lazy_teardown(function() if client then client:close() end if admin_client then admin_client:close() end helpers.stop_kong("vault_ttl_test_cp") helpers.stop_kong("vault_ttl_test_dp") vault:teardown() helpers.unsetenv("KONG_LUA_PATH_OVERRIDE") end) it("updates plugin config references while initial with an invalid string (backend: #" .. vault.name .. ")", function() helpers.wait_for_all_config_update({ forced_admin_port = 9001, forced_proxy_port = 9002, }) -- Switch to vault referenced key field local res = assert(admin_client:patch("/certificates/"..certificate.id, { body = { key = fmt("{vault://%s/%s?ttl=%s}", vault.prefix, secret, 2), cert = ssl_fixtures.cert_client, }, headers = { ["Content-Type"] = "application/json", }, })) assert.res_status(200, res) helpers.wait_for_all_config_update({ forced_admin_port = 9001, forced_proxy_port = 9002, }) -- Update secret value to an invalid key format vault:update_secret(secret, "an invalid string", { ttl = 2 }) -- Wait until the invalid key is being cached assert.with_timeout(7) .with_step(0.5) .ignore_exceptions(true) .eventually(function() helpers.clean_logfile("vault_ttl_test_dp/logs/error.log") local res = client:get("/tls", { headers = { host = "example.com", }, timeout = 2, }) local body = assert.res_status(400, res) assert.matches("No required SSL certificate was sent", body) assert.logfile("vault_ttl_test_dp/logs/error.log").has.line( 'failed to get from node cache: could not parse PEM private key:', true) return true end).is_truthy("Invalid certificate being cached") -- Update secret value and let cert be correct vault:update_secret(secret, ssl_fixtures.key_client, { ttl = 2 }) assert.with_timeout(7) .with_step(0.5) .ignore_exceptions(true) .eventually(function() local res = client:get("/tls", { headers = { host = "example.com", }, timeout = 2, }) local body = assert.res_status(200, res) assert.matches("it works", body) return true end).is_truthy("Expected certificate being refreshed") end) end) end) end -- each vault backend end -- each strategy