LLM Evaluations: LLM eval datasets, graders, metrics, and regression gates.
[
{
"error_code": "SYNTH-LLM-EVALUATIONS-001",
"description": "Synthetic LLM eval datasets, graders, metrics, and regression gates scenario 001: a topic-specific operation failed validation or execution.",
"programmatic_fix": "Validate the LLM eval datasets, graders, metrics, and regression gates inputs and configuration, handle SYNTH-LLM-EVALUATIONS-001 explicitly, and retry only when the operation is idempotent."
},
{
"error_code": "SYNTH-LLM-EVALUATIONS-002",
"description": "Synthetic LLM eval datasets, graders, metrics, and regression gates scenario 002: a topic-specific operation failed validation or execution.",
"programmatic_fix": "Validate the LLM eval datasets, graders, metrics, and regression gates inputs and configuration, handle SYNTH-LLM-EVALUATIONS-002 explicitly, and retry only when the operation is idempotent."
},
{
"error_code": "SYNTH-LLM-EVALUATIONS-003",
"description": "Synthetic LLM eval datasets, graders, metrics, and regression gates scenario 003: a topic-specific operation failed validation or execution.",
"programmatic_fix": "Validate the LLM eval datasets, graders, metrics, and regression gates inputs and configuration, handle SYNTH-LLM-EVALUATIONS-003 explicitly, and retry only when the operation is idempotent."
}
]