# LICENSE HEADER MANAGED BY add-license-header # # Copyright 2018 Kornia Team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import pytest import torch from kornia.feature.lightglue import ( LearnableFourierPositionalEncoding, LightGlue, TokenConfidence, apply_cached_rotary_emb, normalize_keypoints, pad_to_length, rotate_half, ) from testing.base import BaseTester # --------------------------------------------------------------------------- # Pure function tests # --------------------------------------------------------------------------- def test_lightglue_empty_after_pruning(): model = LightGlue(features="superpoint", width_confidence=0.99) model.eval() data = { "image0": { "keypoints": torch.empty(1, 0, 2), "descriptors": torch.empty(1, 0, 256), "image_size": torch.tensor([[640, 480]]), }, "image1": { "keypoints": torch.empty(1, 0, 2), "descriptors": torch.empty(1, 0, 256), "image_size": torch.tensor([[640, 480]]), }, } with torch.no_grad(): out = model(data) assert out["matches0"].shape == (1, 0) assert out["matches1"].shape == (1, 0) assert out["matching_scores0"].shape == (1, 0) assert out["matching_scores1"].shape == (1, 0) def test_lightglue_pruning_removes_all(): model = LightGlue(features="superpoint", width_confidence=0.0) model.eval() B, M, D = 1, 8, 256 data = { "image0": { "keypoints": torch.rand(B, M, 2), "descriptors": torch.rand(B, M, D), "image_size": torch.tensor([[640, 480]]), }, "image1": { "keypoints": torch.rand(B, M, 2), "descriptors": torch.rand(B, M, D), "image_size": torch.tensor([[640, 480]]), }, } with torch.no_grad(): out = model(data) assert "matches0" in out assert "matches1" in out assert out["matches0"].shape == (B, M) assert out["matches1"].shape == (B, M) def test_normalize_keypoints_smoke(): kpts = torch.zeros(1, 5, 2) size = torch.tensor([[100, 200]]) out = normalize_keypoints(kpts, size) assert out.shape == kpts.shape def test_normalize_keypoints_center(): size = torch.tensor([[100, 100]]) kpts = size.float().unsqueeze(0) / 2 # center pixel out = normalize_keypoints(kpts, size) assert torch.allclose(out, torch.zeros_like(out), atol=1e-6) def test_normalize_keypoints_range(): size = torch.tensor([[128, 128]]) kpts = torch.rand(1, 20, 2) * 128 out = normalize_keypoints(kpts, size) assert out.min() >= -1.0 - 1e-3 assert out.max() <= 1.0 + 1e-3 def test_pad_to_length_no_pad(): x = torch.rand(1, 10, 64) y, mask = pad_to_length(x, 5) assert y is x assert mask.shape[-2] == 10 def test_pad_to_length_pads(): x = torch.rand(1, 5, 64) y, mask = pad_to_length(x, 10) assert y.shape[-2] == 10 assert mask.shape[-2] == 10 assert mask[..., :5, :].all() assert not mask[..., 5:, :].any() def test_rotate_half_shape(): x = torch.rand(2, 8, 4) out = rotate_half(x) assert out.shape == x.shape def test_rotate_half_double_gives_negation(): x = torch.rand(1, 4, 8) assert torch.allclose(rotate_half(rotate_half(x)), -x, atol=1e-6) def test_apply_cached_rotary_emb_shape(): B, N, D = 1, 10, 16 freqs = torch.rand(2, B, 1, N, D) t = torch.rand(B, 1, N, D) out = apply_cached_rotary_emb(freqs, t) assert out.shape == t.shape # --------------------------------------------------------------------------- # Module tests # --------------------------------------------------------------------------- class TestLearnableFourierPositionalEncoding(BaseTester): def test_smoke(self, device, dtype): enc = LearnableFourierPositionalEncoding(2, 64).to(device, dtype) x = torch.rand(1, 1, 10, 2, device=device, dtype=dtype) out = enc(x) assert out.shape[0] == 2 # cosines/sines stack def test_cardinality(self, device, dtype): M, dim = 2, 32 enc = LearnableFourierPositionalEncoding(M, dim).to(device, dtype) B, N = 2, 15 x = torch.rand(B, 1, N, M, device=device, dtype=dtype) out = enc(x) assert out.shape[-1] == dim def test_gradcheck(self, device): enc = LearnableFourierPositionalEncoding(2, 32).to(device, torch.float64) x = torch.rand(1, 1, 5, 2, device=device, dtype=torch.float64, requires_grad=True) self.gradcheck(enc, (x,)) def test_dynamo(self, device, dtype, torch_optimizer): enc = LearnableFourierPositionalEncoding(2, 32).to(device, dtype) x = torch.rand(1, 1, 5, 2, device=device, dtype=dtype) op = torch_optimizer(enc) self.assert_close(op(x), enc(x)) def test_exception(self, device, dtype): pass def test_module(self, device, dtype): enc = LearnableFourierPositionalEncoding(2, 32).to(device, dtype) x = torch.rand(1, 1, 5, 2, device=device, dtype=dtype) assert enc(x).shape[0] == 2 class TestTokenConfidence(BaseTester): def test_smoke(self, device, dtype): tc = TokenConfidence(64).to(device, dtype) d0 = torch.rand(1, 10, 64, device=device, dtype=dtype) d1 = torch.rand(1, 8, 64, device=device, dtype=dtype) s0, s1 = tc(d0, d1) assert s0.shape == (1, 10) assert s1.shape == (1, 8) def test_cardinality(self, device, dtype): tc = TokenConfidence(32).to(device, dtype) B, M, N = 2, 12, 15 d0 = torch.rand(B, M, 32, device=device, dtype=dtype) d1 = torch.rand(B, N, 32, device=device, dtype=dtype) s0, s1 = tc(d0, d1) assert s0.shape == (B, M) assert s1.shape == (B, N) def test_scores_in_range(self, device, dtype): tc = TokenConfidence(32).to(device, dtype) d0 = torch.rand(1, 20, 32, device=device, dtype=dtype) d1 = torch.rand(1, 20, 32, device=device, dtype=dtype) s0, s1 = tc(d0, d1) assert (s0 >= 0).all() and (s0 <= 1).all() assert (s1 >= 0).all() and (s1 <= 1).all() def test_gradcheck(self, device): pass # TokenConfidence uses detach() on inputs; not differentiable w.r.t. inputs def test_dynamo(self, device, dtype, torch_optimizer): tc = TokenConfidence(32).to(device, dtype) d0 = torch.rand(1, 10, 32, device=device, dtype=dtype) d1 = torch.rand(1, 8, 32, device=device, dtype=dtype) op = torch_optimizer(tc) s0_jit, s1_jit = op(d0, d1) s0, s1 = tc(d0, d1) self.assert_close(s0_jit, s0) self.assert_close(s1_jit, s1) def test_exception(self, device, dtype): pass def test_module(self, device, dtype): tc = TokenConfidence(32).to(device, dtype) d0 = torch.rand(1, 5, 32, device=device, dtype=dtype) d1 = torch.rand(1, 5, 32, device=device, dtype=dtype) s0, _ = tc(d0, d1) assert s0.shape == (1, 5) # --------------------------------------------------------------------------- # LightGlue (no pretrained weights) tests # --------------------------------------------------------------------------- def _make_lightglue(device, dtype, input_dim=64, n_layers=2): """Instantiate a small LightGlue with random weights (features=None).""" return ( LightGlue( features=None, input_dim=input_dim, descriptor_dim=64, n_layers=n_layers, num_heads=4, depth_confidence=-1, width_confidence=-1, flash=False, ) .to(device, dtype) .eval() ) def _make_data(device, dtype, B=1, M=20, N=15, H=64, W=64, D=64): """Build a minimal data dict for LightGlue forward.""" return { "image0": { "keypoints": torch.rand(B, M, 2, device=device, dtype=dtype) * torch.tensor([W, H], device=device, dtype=dtype), "descriptors": torch.rand(B, M, D, device=device, dtype=dtype), "image_size": torch.tensor([[W, H]], device=device, dtype=dtype).expand(B, -1), }, "image1": { "keypoints": torch.rand(B, N, 2, device=device, dtype=dtype) * torch.tensor([W, H], device=device, dtype=dtype), "descriptors": torch.rand(B, N, D, device=device, dtype=dtype), "image_size": torch.tensor([[W, H]], device=device, dtype=dtype).expand(B, -1), }, } class TestLightGlue(BaseTester): def test_smoke(self, device, dtype): if dtype == torch.float16: pytest.skip("LightGlue requires float32 or float64") lg = _make_lightglue(device, dtype) data = _make_data(device, dtype) with torch.no_grad(): out = lg(data) assert "matches0" in out assert "matching_scores0" in out def test_cardinality(self, device, dtype): if dtype == torch.float16: pytest.skip("LightGlue requires float32 or float64") B, M, N = 1, 20, 15 lg = _make_lightglue(device, dtype) data = _make_data(device, dtype, B=B, M=M, N=N) with torch.no_grad(): out = lg(data) assert out["matches0"].shape == (B, M) assert out["matches1"].shape == (B, N) assert out["matching_scores0"].shape == (B, M) def test_image_tensor_input(self, device, dtype): """Accept image tensor instead of image_size.""" if dtype == torch.float16: pytest.skip("LightGlue requires float32 or float64") B, M, N, H, W, D = 1, 10, 8, 32, 32, 64 lg = _make_lightglue(device, dtype) data = { "image0": { "keypoints": torch.rand(B, M, 2, device=device, dtype=dtype) * W, "descriptors": torch.rand(B, M, D, device=device, dtype=dtype), "image": torch.rand(B, 3, H, W, device=device, dtype=dtype), }, "image1": { "keypoints": torch.rand(B, N, 2, device=device, dtype=dtype) * W, "descriptors": torch.rand(B, N, D, device=device, dtype=dtype), "image": torch.rand(B, 3, H, W, device=device, dtype=dtype), }, } with torch.no_grad(): out = lg(data) assert "matches0" in out def test_matches_are_valid_indices(self, device, dtype): if dtype == torch.float16: pytest.skip("LightGlue requires float32 or float64") B, M, N = 1, 30, 25 lg = _make_lightglue(device, dtype) data = _make_data(device, dtype, B=B, M=M, N=N) with torch.no_grad(): out = lg(data) m0 = out["matches0"] # (B, M), values in [-1, N-1] assert (m0 >= -1).all() assert (m0 < N).all() def test_scores_in_range(self, device, dtype): if dtype == torch.float16: pytest.skip("LightGlue requires float32 or float64") lg = _make_lightglue(device, dtype) data = _make_data(device, dtype) with torch.no_grad(): out = lg(data) scores = out["matching_scores0"] assert (scores >= 0).all() assert (scores <= 1).all() def test_exception_missing_key(self, device, dtype): if dtype == torch.float16: pytest.skip("LightGlue requires float32 or float64") lg = _make_lightglue(device, dtype) with pytest.raises(Exception): lg({"image0": {}}) def test_exception_wrong_features(self, device, dtype): with pytest.raises(Exception): LightGlue(features="nonexistent_feature_type") def test_gradcheck(self, device): pass # LightGlue uses detach() on descriptors; not differentiable end-to-end def test_dynamo(self, device, dtype, torch_optimizer): pass # LightGlue uses dynamic control flow; not torch.compile compatible def test_module(self, device, dtype): if dtype == torch.float16: pytest.skip("LightGlue requires float32 or float64") lg = _make_lightglue(device, dtype) data = _make_data(device, dtype) with torch.no_grad(): out = lg(data) assert isinstance(out, dict) @pytest.mark.slow def test_pretrained_smoke(self, device): """Instantiating with real feature type downloads and loads weights.""" lg = LightGlue(features="disk", depth_confidence=-1, width_confidence=-1).to(device).eval() B, M, N, D = 1, 50, 50, 128 data = _make_data(device, torch.float32, B=B, M=M, N=N, D=D) with torch.no_grad(): out = lg(data) assert "matches0" in out