Eval | Mixtrain Docs

from mixtrain import Eval

Constructor

Eval(name: str)

Creates a reference to an existing evaluation. This is a lazy operation - no API call is made until you access properties or call methods.

Parameter	Type	Description
`name`	`str`	Evaluation name

eval = Eval("image-quality-eval")

Properties

Property	Type	Description
`name`	`str`	Evaluation name
`description`	`str`	Evaluation description
`config`	`dict`	Evaluation configuration
`status`	`str`	Status: `"pending"`, `"running"`, `"completed"`, `"failed"`
`metadata`	`dict`	Full metadata dictionary (cached)

Methods

update()

Update evaluation metadata, config, or status.

eval.update(
    status: str = None,
    config: dict = None,
    description: str = None,
    **kwargs
) -> None

Parameter	Type	Description
`status`	`str`	New status
`config`	`dict`	Updated configuration (merged with existing)
`description`	`str`	New description

eval.update(
    status="completed",
    config={
        "results": {
            "flux-pro": {"quality": 0.92},
            "stable-diffusion-xl": {"quality": 0.88}
        }
    }
)

delete()

Delete the evaluation.

eval.delete() -> None

refresh()

Clear cached data.

eval.refresh() -> None

Class Methods

Eval.from_dataset()

Create an evaluation from a dataset, automatically using its column types for the comparison config.

Eval.from_dataset(
    dataset: Dataset | str,
    name: str = None,
    description: str = None,
    columns: list[str] = None
) -> Eval

Parameter	Type	Description
`dataset`	`Dataset \| str`	Dataset instance or name
`name`	`str`	Evaluation name (defaults to `"{dataset}-eval"`)
`description`	`str`	Optional description (auto-generated if omitted)
`columns`	`list[str]`	Columns to include and their order (defaults to all typed columns)

Returns: Eval

# Use all typed columns from the dataset
eval = Eval.from_dataset("image-gen-results")

# Select specific columns and their order
eval = Eval.from_dataset(
    "image-gen-results",
    name="flux-vs-sdxl",
    columns=["prompt", "flux_output", "sdxl_output"]
)

# From a Dataset instance
ds = Dataset("image-gen-results")
eval = Eval.from_dataset(ds)

Eval.create()

Create an evaluation with a manual configuration.

Eval.create(
    name: str,
    config: dict,
    description: str = None
) -> Eval

Parameter	Type	Description
`name`	`str`	Evaluation name
`config`	`dict`	Configuration with `datasets` array defining columns to display
`description`	`str`	Optional description

Returns: Eval

eval = Eval.create(
    name="image-quality-eval",
    config={
        "datasets": [
            {"tableName": "results", "columnName": "prompt", "dataType": "text"},
            {"tableName": "results", "columnName": "flux_output", "dataType": "image"},
            {"tableName": "results", "columnName": "sdxl_output", "dataType": "image"},
        ]
    },
    description="Compare image generation models"
)

Eval.exists()

Check if an evaluation exists.

Eval.exists(name: str) -> bool

Parameter	Type	Description
`name`	`str`	Evaluation name to check

Returns: bool - True if the evaluation exists, False otherwise

if not Eval.exists("my-eval"):
    Eval.from_dataset("results", name="my-eval")

list_evals()

List all evaluations in the workspace.

from mixtrain import list_evals

evals = list_evals()
for e in evals:
    print(f"{e.name}: {e.status}")

On this page