ludwig-ai--ludwig
wehub-resource-sync
593b94c120
pytest / Unit Tests (push) Has been cancelled
pytest / Integration (integration_tests_a) (push) Has been cancelled
pytest / Integration (integration_tests_b) (push) Has been cancelled
pytest / Integration (integration_tests_c) (push) Has been cancelled
pytest / Integration (integration_tests_d) (push) Has been cancelled
pytest / Integration (integration_tests_e) (push) Has been cancelled
pytest / Integration (integration_tests_f) (push) Has been cancelled
pytest / Integration (integration_tests_g) (push) Has been cancelled
pytest / Integration (integration_tests_h) (push) Has been cancelled
pytest / Integration (integration_tests_i) (push) Has been cancelled
pytest / Integration (integration_tests_j) (push) Has been cancelled
pytest / Distributed (distributed_a) (push) Has been cancelled
pytest / Distributed (distributed_b) (push) Has been cancelled
pytest / Distributed (distributed_c) (push) Has been cancelled
pytest / Distributed (distributed_d) (push) Has been cancelled
pytest / Distributed (distributed_e) (push) Has been cancelled
pytest / Distributed (distributed_f) (push) Has been cancelled
pytest / Minimal Install (push) Has been cancelled
pytest / Event File (push) Has been cancelled
pytest (slow) / py-slow (push) Has been cancelled
Publish JSON Schema / publish-schema (push) Has been cancelled
60 行
2.5 KiB
Python
60 行
2.5 KiB
Python
from ludwig.api_annotations import DeveloperAPI
|
|
from ludwig.constants import RANDOM
|
|
from ludwig.schema import utils as schema_utils
|
|
from ludwig.schema.metadata import PREPROCESSING_METADATA
|
|
from ludwig.schema.split import BaseSplitConfig, SplitDataclassField
|
|
|
|
|
|
@DeveloperAPI
|
|
class PreprocessingConfig(schema_utils.LudwigBaseConfig):
|
|
"""Global preprocessing config is a dataclass that configures the parameters used for global preprocessing."""
|
|
|
|
sample_ratio: float = schema_utils.NonNegativeFloat(
|
|
default=1.0,
|
|
description="The ratio of the dataset to use. For instance, if 0.5, half of the dataset provided will be used.",
|
|
parameter_metadata=PREPROCESSING_METADATA["sample_ratio"],
|
|
)
|
|
|
|
sample_size: float = schema_utils.NonNegativeInteger(
|
|
default=None,
|
|
allow_none=True,
|
|
description="The maximum number of samples from the dataset to use. Cannot be set if sample_ratio is set to be "
|
|
"< 1.0. If sample_ratio is set to 1.0, this will override the number of samples to used.",
|
|
parameter_metadata=PREPROCESSING_METADATA["sample_size"],
|
|
)
|
|
|
|
oversample_minority: float = schema_utils.NonNegativeFloat(
|
|
default=None,
|
|
allow_none=True,
|
|
description="If not None, the minority class will be oversampled to reach the specified ratio respective to "
|
|
"the majority class. ",
|
|
parameter_metadata=PREPROCESSING_METADATA["oversample_minority"],
|
|
)
|
|
|
|
undersample_majority: float = schema_utils.NonNegativeFloat(
|
|
default=None,
|
|
allow_none=True,
|
|
description="If not None, the majority class will be undersampled to reach the specified ratio respective "
|
|
"to the minority class. ",
|
|
parameter_metadata=PREPROCESSING_METADATA["undersample_majority"],
|
|
)
|
|
|
|
split: BaseSplitConfig = SplitDataclassField(
|
|
default=RANDOM,
|
|
)
|
|
|
|
global_max_sequence_length: int = schema_utils.PositiveInteger(
|
|
default=None,
|
|
allow_none=True,
|
|
description="Specifically for LLMs. This is the maximum length of the input sequence going into the model's "
|
|
"forward pass during training. Sequences will be truncated to this length after merging inputs and targets. "
|
|
"If not set, the total length of the merged input and target token sequences will be used.",
|
|
parameter_metadata=PREPROCESSING_METADATA["global_max_sequence_length"],
|
|
)
|
|
|
|
|
|
@DeveloperAPI
|
|
class PreprocessingField(schema_utils.NestedConfigField):
|
|
def __init__(self):
|
|
super().__init__(PreprocessingConfig)
|