cleanlab--cleanlab
OrdoAbChao
a6d1319327
Preleminary work to ensure that default task (classification) will work smoothly. Minimal implementation of label issue manager for regression - issue_finder.py list_possible_issue_types, list_default_issue_types no longer a static methods as they depend on task and modification of get_available_issue_types for regression * DataIssues with strategy pattern - moving list_possible_issue_types and list_default_issue_types from the IssueFinder class to cleanlab/datalab/internal/issue_manager_factory.py * make different strategies for getting available issue types add test class for the IssueFinder when the task is regression * apply black formatter * avoid mapping labels column for regression in Datalab * Pass in features to LabelIssueManager for regression --------- Co-authored-by: Elías Snorrason <eliassno@gmail.com>
110 行
2.9 KiB
Python
110 行
2.9 KiB
Python
import numpy as np
|
|
import pandas as pd
|
|
import pytest
|
|
|
|
from cleanlab.datalab.internal.issue_manager import IssueManager
|
|
from cleanlab.datalab.internal.issue_manager_factory import (
|
|
REGISTRY,
|
|
register,
|
|
)
|
|
|
|
|
|
class TestCustomIssueManager:
|
|
@pytest.mark.parametrize(
|
|
"score",
|
|
[0, 0.5, 1],
|
|
ids=["zero", "positive_float", "one"],
|
|
)
|
|
def test_make_summary_with_score(self, custom_issue_manager, score):
|
|
summary = custom_issue_manager.make_summary(score=score)
|
|
|
|
expected_summary = pd.DataFrame(
|
|
{
|
|
"issue_type": [custom_issue_manager.issue_name],
|
|
"score": [score],
|
|
}
|
|
)
|
|
assert pd.testing.assert_frame_equal(summary, expected_summary) is None
|
|
|
|
@pytest.mark.parametrize(
|
|
"score",
|
|
[-0.3, 1.5, np.nan, np.inf, -np.inf],
|
|
ids=["negative_float", "greater_than_one", "nan", "inf", "negative_inf"],
|
|
)
|
|
def test_make_summary_invalid_score(self, custom_issue_manager, score):
|
|
with pytest.raises(ValueError):
|
|
custom_issue_manager.make_summary(score=score)
|
|
|
|
|
|
def test_register_custom_issue_manager(monkeypatch):
|
|
import io
|
|
import sys
|
|
|
|
assert "foo" not in REGISTRY
|
|
|
|
class Foo(IssueManager):
|
|
issue_name = "foo"
|
|
|
|
def find_issues(self):
|
|
pass
|
|
|
|
Foo = register(Foo)
|
|
|
|
assert REGISTRY["classification"].get("foo") == Foo
|
|
|
|
# Reregistering should overwrite the existing class, put print a warning
|
|
|
|
monkeypatch.setattr("sys.stdout", io.StringIO())
|
|
|
|
class NewFoo(IssueManager):
|
|
issue_name = "foo"
|
|
|
|
def find_issues(self):
|
|
pass
|
|
|
|
NewFoo = register(NewFoo)
|
|
|
|
assert REGISTRY["classification"].get("foo") == NewFoo
|
|
assert all(
|
|
[
|
|
text in sys.stdout.getvalue()
|
|
for text in [
|
|
"Warning: Overwriting existing issue manager foo with ",
|
|
"NewFoo",
|
|
" for task classification.",
|
|
]
|
|
]
|
|
), "Should print a warning"
|
|
|
|
# Reregistering for task should overwrite the existing class, put print a warning
|
|
class NewerFoo(IssueManager):
|
|
issue_name = "label"
|
|
|
|
def find_issues(self):
|
|
pass
|
|
|
|
NewerFoo = register(NewerFoo, task="classification")
|
|
|
|
assert REGISTRY["classification"].get("label") == NewerFoo
|
|
assert all(
|
|
[
|
|
text in sys.stdout.getvalue()
|
|
for text in [
|
|
"Warning: Overwriting existing issue manager label with ",
|
|
"NewerFoo",
|
|
" for task classification.",
|
|
]
|
|
]
|
|
), "Should print a warning"
|
|
|
|
# Registering any issue manager for another task is permitted
|
|
class Bar(IssueManager):
|
|
issue_name = "bar"
|
|
|
|
def find_issues(self):
|
|
pass
|
|
|
|
Bar = register(Bar, task="regression")
|
|
|
|
assert REGISTRY["regression"].get("bar") == Bar
|