local helpers = require "spec.helpers" -- using the full path so that we don't have to modify package.path in -- this context local test_vault = require "spec.fixtures.custom_vaults.kong.vaults.test" local CUSTOM_VAULTS = "./spec/fixtures/custom_vaults" local CUSTOM_PLUGINS = "./spec/fixtures/custom_plugins" local LUA_PATH = CUSTOM_VAULTS .. "/?.lua;" .. CUSTOM_VAULTS .. "/?/init.lua;" .. CUSTOM_PLUGINS .. "/?.lua;" .. CUSTOM_PLUGINS .. "/?/init.lua;;" local DUMMY_HEADER = "Dummy-Plugin" local fmt = string.format local json = require "cjson" --- A vault test harness is a driver for vault backends, which implements --- all the necessary glue for initializing a vault backend and performing --- secret read/write operations. --- --- All functions defined here are called as "methods" (e.g. harness:fn()), so --- it is permitted to keep state on the harness object (self). --- ---@class vault_test_harness --- ---@field name string --- --- this table is passed directly to kong.db.vaults:insert() ---@field config table --- --- create_secret() is called once per test run for a given secret ---@field create_secret fun(self: vault_test_harness, secret: string, value: string, opts?: table) --- --- update_secret() may be called more than once per test run for a given secret ---@field update_secret fun(self: vault_test_harness, secret: string, value: string, opts?: table) --- --- setup() is called before kong is started and before any DB entities --- have been created and is best used for things like validating backend --- credentials and establishing a connection to a backend ---@field setup fun(self: vault_test_harness) --- --- teardown() is exactly what you'd expect ---@field teardown fun(self: vault_test_harness) --- --- fixtures() output is passed directly to `helpers.start_kong()` ---@field fixtures fun(self: vault_test_harness):table|nil --- --- pause() is exactly what you'd expect ---@field pause fun(self: vault_test_harness) --- --- ---@field prefix string # generated by the test suite ---@field host string # generated by the test suite ---@type vault_test_harness[] local VAULTS = { { name = "test", config = { default_value = "DEFAULT", default_value_ttl = 1, }, create_secret = function(self, _, value) -- Currently, create_secret is called _before_ starting Kong. -- -- This means our backend won't be available yet because it is -- piggy-backing on Kong as an HTTP mock fixture. -- -- We can, however, inject a default value into our configuration. self.config.default_value = value end, update_secret = function(_, secret, value, opts) return test_vault.client.put(secret, value, opts) end, delete_secret = function(_, secret) return test_vault.client.delete(secret) end, fixtures = function() return { http_mock = { test_vault = test_vault.http_mock, } } end, pause = function(_) return test_vault.client.pause() end, }, } local noop = function(...) end for _, vault in ipairs(VAULTS) do -- fill out some values that we'll use in route/service/plugin config vault.prefix = vault.name .. "-ttl-test" vault.host = vault.name .. ".vault-ttl.test" -- ...and fill out non-required methods vault.setup = vault.setup or noop vault.teardown = vault.teardown or noop vault.fixtures = vault.fixtures or noop vault.pause = vault.pause or noop end for _, strategy in helpers.each_strategy() do for _, vault in ipairs(VAULTS) do describe("vault resurrect_ttl and rotation (#" .. strategy .. ") #" .. vault.name, function() local client local secret = "my-secret" local function http_get(path) path = path or "/" local res = client:get(path, { headers = { host = assert(vault.host), }, }) assert.response(res).has.status(200) return res end lazy_setup(function() helpers.setenv("KONG_LUA_PATH_OVERRIDE", LUA_PATH) helpers.setenv("KONG_VAULT_ROTATION_INTERVAL", "1") vault:setup() vault:create_secret(secret, "init") local bp = helpers.get_db_utils(strategy, { "vaults", "routes", "services", "plugins" }, { "dummy" }, { vault.name }) assert(bp.vaults:insert({ name = vault.name, prefix = vault.prefix, config = vault.config, })) local route = assert(bp.routes:insert({ name = vault.host, hosts = { vault.host }, paths = { "/" }, service = assert(bp.services:insert()), })) -- used by the plugin config test case assert(bp.plugins:insert({ name = "dummy", config = { resp_header_value = fmt("{vault://%s/%s?ttl=%d&resurrect_ttl=%d}", vault.prefix, secret, 2, 2), }, route = { id = route.id }, })) assert(helpers.start_kong({ database = strategy, nginx_conf = "spec/fixtures/custom_nginx.template", vaults = vault.name, plugins = "dummy", log_level = "info", }, nil, nil, vault:fixtures() )) client = helpers.proxy_client() end) lazy_teardown(function() if client then client:close() end helpers.stop_kong() vault:teardown() helpers.unsetenv("KONG_LUA_PATH_OVERRIDE") end) it("resurrects plugin config references when secret is deleted (backend: #" .. vault.name .. ")", function() local function check_plugin_secret(expect, ttl, leeway) leeway = leeway or 0.25 -- 25% local timeout = ttl + (ttl * leeway) assert .with_timeout(timeout) .with_step(0.5) .eventually(function() local res = http_get("/") local value if expect == "" then value = res.headers[DUMMY_HEADER] or "" if value == "" then return true end else value = assert.response(res).has.header(DUMMY_HEADER) if value == expect then return true end end return nil, { expected = expect, got = value } end) .is_truthy("expected plugin secret to be updated to '" .. tostring(expect) .. "' " .. "within " .. tostring(timeout) .. " seconds") end vault:update_secret(secret, "old", { ttl = 2, resurrect_ttl = 2 }) check_plugin_secret("old", 5) vault:delete_secret(secret) ngx.sleep(2.5) check_plugin_secret("old", 5) check_plugin_secret("", 5) end) end) describe("#multiworker vault resurrect_ttl and rotation (#" .. strategy .. ") #" .. vault.name, function() local client, admin_client local secret = "my-secret" lazy_setup(function() helpers.setenv("KONG_LUA_PATH_OVERRIDE", LUA_PATH) helpers.setenv("KONG_VAULT_ROTATION_INTERVAL", "1") vault:setup() vault:create_secret(secret, "init") local bp = helpers.get_db_utils(strategy, { "vaults", "routes", "services", "plugins" }, { "dummy" }, { vault.name }) assert(bp.vaults:insert({ name = vault.name, prefix = vault.prefix, config = vault.config, })) local route = assert(bp.routes:insert({ name = vault.host, hosts = { vault.host }, paths = { "/" }, service = assert(bp.services:insert()), })) assert(bp.plugins:insert({ name = "post-function", config = { access = {fmt([[ local value, err = kong.vault.get("{vault://%s/%s?ttl=%d&resurrect_ttl=%d}") if value then kong.response.exit(200, {["value"]=value, ["pid"]=ngx.worker.pid()}, {["Content-Type"]="application/json"}) end ]], vault.prefix, secret, 2, 5),} }, route = { id = route.id }, })) assert(helpers.start_kong({ database = strategy, nginx_conf = "spec/fixtures/custom_nginx.template", vaults = vault.name, plugins = "post-function", log_level = "debug", dedicated_config_processing = false, -- nginx_worker_processes = 2, nginx_main_worker_processes = 2, }, nil, nil, vault:fixtures() )) client = helpers.proxy_client() admin_client = helpers.admin_client() end) lazy_teardown(function() if client then client:close() end if admin_client then admin_client:close() end helpers.stop_kong() vault:teardown() helpers.unsetenv("KONG_LUA_PATH_OVERRIDE") end) it("resurrects secret value from shared dict when secret is deleted (backend: #" .. vault.name .. ")", function() -- fetch all worker pids local status_ret = admin_client:get("/") local body = assert.res_status(200, status_ret) local json_body = json.decode(body) assert.truthy(json_body) local worker_pids = json_body.pids.workers assert.truthy(#worker_pids == 2) local worker_secret_hits = {} for _, worker_pid in ipairs(worker_pids) do worker_secret_hits[tostring(worker_pid)] = false end vault:update_secret(secret, "old", { ttl = 2, resurrect_ttl = 5 }) -- trigger post-function in one of the workers local res = client:get("/", {headers = {host = assert(vault.host)}}) local body = assert.res_status(200, res) local json_body = json.decode(body) assert.same("old", json_body.value) worker_secret_hits[tostring(json_body.pid)] = true vault:pause() -- let ttl pass and try to trigger post-function in all workers -- check all of them can resurrect the secret from shared dict ngx.sleep(3) assert.with_timeout(5).with_step(0.1).eventually( function() -- avoid connection reuse so that we can hit all workers local new_client = helpers.proxy_client() local res = new_client:get("/", {headers = {host = assert(vault.host)}}) local body = assert.res_status(200, res) local json_body = json.decode(body) new_client:close() assert.same("old", json_body.value) worker_secret_hits[tostring(json_body.pid)] = true for k, v in pairs(worker_secret_hits) do if not v then return false, "worker pid " .. k .. " did not hit the secret" end end return true end ).is_truthy("expected all workers to resurrect the secret from shared dict") end) end) end -- each vault backend end -- each strategy