ludwig-ai--ludwig
wehub-resource-sync
593b94c120
pytest / Unit Tests (push) Has been cancelled
pytest / Integration (integration_tests_a) (push) Has been cancelled
pytest / Integration (integration_tests_b) (push) Has been cancelled
pytest / Integration (integration_tests_c) (push) Has been cancelled
pytest / Integration (integration_tests_d) (push) Has been cancelled
pytest / Integration (integration_tests_e) (push) Has been cancelled
pytest / Integration (integration_tests_f) (push) Has been cancelled
pytest / Integration (integration_tests_g) (push) Has been cancelled
pytest / Integration (integration_tests_h) (push) Has been cancelled
pytest / Integration (integration_tests_i) (push) Has been cancelled
pytest / Integration (integration_tests_j) (push) Has been cancelled
pytest / Distributed (distributed_a) (push) Has been cancelled
pytest / Distributed (distributed_b) (push) Has been cancelled
pytest / Distributed (distributed_c) (push) Has been cancelled
pytest / Distributed (distributed_d) (push) Has been cancelled
pytest / Distributed (distributed_e) (push) Has been cancelled
pytest / Distributed (distributed_f) (push) Has been cancelled
pytest / Minimal Install (push) Has been cancelled
pytest / Event File (push) Has been cancelled
pytest (slow) / py-slow (push) Has been cancelled
Publish JSON Schema / publish-schema (push) Has been cancelled
21 行
1.2 KiB
Python
21 行
1.2 KiB
Python
def check_global_max_sequence_length_fits_prompt_template(metadata, global_preprocessing_parameters):
|
|
"""Checks that the prompt template fits within the global max sequence length."""
|
|
|
|
if (
|
|
"global_max_sequence_length" in global_preprocessing_parameters
|
|
and global_preprocessing_parameters["global_max_sequence_length"] is not None
|
|
):
|
|
for _feature_name, feature_metadata in metadata.items():
|
|
if (
|
|
"prompt_template_num_tokens" in feature_metadata
|
|
and feature_metadata["prompt_template_num_tokens"]
|
|
> global_preprocessing_parameters["global_max_sequence_length"]
|
|
):
|
|
raise ValueError(
|
|
f"The prompt contains ({feature_metadata['prompt_template_num_tokens']}) tokens, which is more "
|
|
f"than the the global_max_sequence_length "
|
|
f"({global_preprocessing_parameters['global_max_sequence_length']}), which will remove all unique "
|
|
"information. Shorten the prompt, or increase the global max sequence length to > "
|
|
f"({feature_metadata['prompt_template_num_tokens']}) to include the full prompt."
|
|
)
|