cleanlab--cleanlab
OrdoAbChao
a6d1319327
Preleminary work to ensure that default task (classification) will work smoothly. Minimal implementation of label issue manager for regression - issue_finder.py list_possible_issue_types, list_default_issue_types no longer a static methods as they depend on task and modification of get_available_issue_types for regression * DataIssues with strategy pattern - moving list_possible_issue_types and list_default_issue_types from the IssueFinder class to cleanlab/datalab/internal/issue_manager_factory.py * make different strategies for getting available issue types add test class for the IssueFinder when the task is regression * apply black formatter * avoid mapping labels column for regression in Datalab * Pass in features to LabelIssueManager for regression --------- Co-authored-by: Elías Snorrason <eliassno@gmail.com>
103 行
3.5 KiB
Python
103 行
3.5 KiB
Python
from unittest.mock import Mock, patch
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import pytest
|
|
|
|
from cleanlab import Datalab
|
|
from cleanlab.datalab.internal.report import Reporter
|
|
|
|
|
|
class TestReporter:
|
|
@pytest.fixture
|
|
def lab(self):
|
|
N = 30
|
|
K = 2
|
|
X = np.random.rand(N, K)
|
|
y = np.random.randint(0, K, size=N)
|
|
pred_probs = np.random.rand(N, K)
|
|
lab = Datalab(data={"y": y}, label_name="y")
|
|
lab.find_issues(features=X, pred_probs=pred_probs)
|
|
return lab
|
|
|
|
@pytest.fixture
|
|
def data_issues(self, lab):
|
|
return lab.data_issues
|
|
|
|
@pytest.fixture
|
|
def reporter(self, data_issues):
|
|
return Reporter(data_issues=data_issues, task="classification")
|
|
|
|
def test_init(self, reporter, data_issues):
|
|
assert reporter.data_issues == data_issues
|
|
assert reporter.verbosity == 1
|
|
assert reporter.include_description == True
|
|
assert reporter.show_summary_score == False
|
|
|
|
another_reporter = Reporter(data_issues=data_issues, task="classification", verbosity=2)
|
|
assert another_reporter.verbosity == 2
|
|
|
|
def test_report(self, reporter):
|
|
"""Test that the report method works. It just wraps the get_report method in a print
|
|
statement."""
|
|
mock_get_report = Mock()
|
|
|
|
with patch("builtins.print") as mock_print: # type: ignore
|
|
with patch.object(reporter, "get_report", mock_get_report):
|
|
reporter.report(num_examples=3)
|
|
mock_get_report.assert_called_with(num_examples=3)
|
|
mock_print.assert_called_with(mock_get_report.return_value)
|
|
|
|
@pytest.mark.parametrize("include_description", [True, False])
|
|
def test_get_report(self, reporter, data_issues, include_description, monkeypatch):
|
|
"""Test that the report method works. Assuming we have two issue managers, each should add
|
|
their section to the report."""
|
|
|
|
mock_issue_manager = Mock()
|
|
mock_issue_manager.issue_name = "foo"
|
|
mock_issue_manager.report.return_value = "foo report"
|
|
|
|
class MockIssueManagerFactory:
|
|
@staticmethod
|
|
def from_str(*args, **kwargs):
|
|
return mock_issue_manager
|
|
|
|
monkeypatch.setattr(
|
|
"cleanlab.datalab.internal.report._IssueManagerFactory", MockIssueManagerFactory
|
|
)
|
|
mock_issues = pd.DataFrame(
|
|
{
|
|
"is_foo_issue": [False, True, False, False, False],
|
|
"foo_score": [0.6, 0.2, 0.7, 0.7, 0.8],
|
|
}
|
|
)
|
|
monkeypatch.setattr(data_issues, "issues", mock_issues)
|
|
|
|
mock_issue_summary = pd.DataFrame(
|
|
{
|
|
"issue_type": ["foo"],
|
|
"score": [0.6],
|
|
"num_issues": [1],
|
|
}
|
|
)
|
|
|
|
mock_info = {"foo": {"bar": "baz"}}
|
|
|
|
monkeypatch.setattr(data_issues, "issue_summary", mock_issue_summary)
|
|
|
|
reporter = Reporter(
|
|
data_issues=data_issues,
|
|
task="classification",
|
|
verbosity=0,
|
|
include_description=include_description,
|
|
)
|
|
monkeypatch.setattr(data_issues, "issues", mock_issues, raising=False)
|
|
monkeypatch.setattr(data_issues, "info", mock_info, raising=False)
|
|
|
|
monkeypatch.setattr(
|
|
reporter, "_write_summary", lambda *args, **kwargs: "Here is a lab summary\n\n"
|
|
)
|
|
report = reporter.get_report(num_examples=3)
|
|
expected_report = "\n\n".join(["Here is a lab summary", "foo report"])
|
|
assert report == expected_report
|