Rapidata benchmark

RapidataBenchmark

RapidataBenchmark(
    name: str, id: str, openapi_service: OpenAPIService
)

An instance of a Rapidata benchmark.

Used to interact with a specific benchmark in the Rapidata system, such as retrieving prompts and evaluating models.

Parameters:

Name	Type	Description	Default
name	str	The name that will be used to identify the benchmark on the overview.	required
id	str	The id of the benchmark.	required
openapi_service	OpenAPIService	The OpenAPI service to use to interact with the Rapidata API.	required

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def __init__(self, name: str, id: str, openapi_service: OpenAPIService):
    self.name = name
    self.id = id
    self._openapi_service = openapi_service
    self.__prompts: list[str | None] = []
    self.__english_prompts: list[str | None] = []
    self.__prompt_assets: list[str | None] = []
    self.__leaderboards: list["RapidataLeaderboard"] = []
    self.__identifiers: list[str] = []
    self.__tags: list[list[str]] = []
    self.__structured_tags: list[list[Tag]] = []
    self.__origins: list[Origin | None] = []
    self.__prompt_ids: dict[str, str] = {}
    self.__participants: list[BenchmarkParticipant] = []
    self.__benchmark_page: str = (
        f"https://app.{self._openapi_service.environment}/mri/benchmarks/{self.id}"
    )
    self._prompt_uploader = BenchmarkPromptUploader(id, openapi_service)

prompts property

prompts: list[str | None]

Returns the prompts as originally provided, in the order they were registered.

english_prompts property

english_prompts: list[str | None]

Returns the prompts translated to English, aligned by index with prompts.

The translations are produced server-side, so accessing this after add_prompts triggers a one-off re-fetch of the prompt set.

prompt_assets property

prompt_assets: list[str | None]

Returns the prompt assets that are registered for the benchmark.

tags property

tags: list[list[str]]

Returns the tag values registered for the benchmark, aligned by index with prompts.

This is the flat, values-only view of the tags. It is kept for backwards compatibility — prefer structured_tags for the (value + category) representation.

structured_tags property

structured_tags: list[list[Tag]]

Returns the structured tags registered for the benchmark, aligned by index with prompts.

Each :class:Tag carries a value and an optional category. Tags are used to filter and organize leaderboard results and are NOT shown to the annotators.

origins property

origins: list[Origin | None]

Returns the origin of each prompt, aligned by index with prompts.

A prompt without an origin is represented as None.

leaderboards property

leaderboards: list[RapidataLeaderboard]

Returns the leaderboards that are registered for the benchmark.

participants property

participants: list[BenchmarkParticipant]

Returns the participants that are registered for the benchmark.

add_prompts

add_prompts(
    identifiers: Optional[list[str]] = None,
    prompts: Optional[list[str | None] | list[str]] = None,
    prompt_assets: Optional[
        list[str | None] | list[str]
    ] = None,
    tags: Optional[
        Sequence[Sequence[str | Tag] | None]
    ] = None,
    origins: Optional[Sequence[Origin | str | None]] = None,
) -> None

Adds one or more prompts to the benchmark. Everything is matched up by the indexes of the lists.

prompts or identifiers must be provided, as well as prompts or prompt_assets.

The prompts are uploaded concurrently. A failed upload does not abort the rest: every prompt is attempted, failures are logged, and only the prompts that succeeded are registered.

Parameters:

Name	Type	Description	Default
identifiers	Optional[list[str]]	The identifiers of the prompts/assets/tags that will be used to match up the media. If not provided, it will use the prompts as the identifiers.	None
prompts	Optional[list[str | None] | list[str]]	The prompts that will be registered for the benchmark.	None
prompt_assets	Optional[list[str | None] | list[str]]	The prompt assets that will be registered for the benchmark.	None
tags	Optional[Sequence[Sequence[str | Tag] | None]]	The tags per prompt, used to filter and organize the leaderboard results. They are NOT shown to the users. Each entry is a list of plain strings, a list of :class:Tag (a value plus an optional category), or a mix of both — strings are converted to Tag(value, category=None) internally. None means no tags for that prompt.	None
origins	Optional[Sequence[Origin | str | None]]	The origin of each prompt (e.g. a source dataset). Each entry is a plain string (converted to Origin(source)), an :class:Origin, or None.	None

Example

# Plain strings are all you need when you don't want categories.
benchmark.add_prompts(
    identifiers=["id1", "id2"],
    prompts=["prompt 1", "prompt 2"],
    prompt_assets=["https://assets.rapidata.ai/prompt_1.jpg", "https://assets.rapidata.ai/prompt_2.jpg"],
    tags=[["landscape", "outdoor"], ["portrait"]],
    origins=["coco", "coco"],
)

# Reach for Tag only where you want to group tags by category.
from rapidata import Tag

benchmark.add_prompts(
    identifiers=["id3"],
    prompts=["prompt 3"],
    tags=[[Tag("landscape", category="scene"), "outdoor"]],
)

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def add_prompts(
    self,
    identifiers: Optional[list[str]] = None,
    prompts: Optional[list[str | None] | list[str]] = None,
    prompt_assets: Optional[list[str | None] | list[str]] = None,
    tags: Optional[Sequence[Sequence[str | Tag] | None]] = None,
    origins: Optional[Sequence[Origin | str | None]] = None,
) -> None:
    """
    Adds one or more prompts to the benchmark. Everything is matched up by the
    indexes of the lists.

    prompts or identifiers must be provided, as well as prompts or prompt_assets.

    The prompts are uploaded concurrently. A failed upload does not abort the
    rest: every prompt is attempted, failures are logged, and only the prompts
    that succeeded are registered.

    Args:
        identifiers: The identifiers of the prompts/assets/tags that will be used to match up the media. If not provided, it will use the prompts as the identifiers.
        prompts: The prompts that will be registered for the benchmark.
        prompt_assets: The prompt assets that will be registered for the benchmark.
        tags: The tags per prompt, used to filter and organize the leaderboard results. They are NOT shown to the users. Each entry is a list of plain strings, a list of :class:`Tag` (a `value` plus an optional `category`), or a mix of both — strings are converted to `Tag(value, category=None)` internally. None means no tags for that prompt.
        origins: The origin of each prompt (e.g. a source dataset). Each entry is a plain string (converted to `Origin(source)`), an :class:`Origin`, or None.

    Example:
        ```python
        # Plain strings are all you need when you don't want categories.
        benchmark.add_prompts(
            identifiers=["id1", "id2"],
            prompts=["prompt 1", "prompt 2"],
            prompt_assets=["https://assets.rapidata.ai/prompt_1.jpg", "https://assets.rapidata.ai/prompt_2.jpg"],
            tags=[["landscape", "outdoor"], ["portrait"]],
            origins=["coco", "coco"],
        )

        # Reach for Tag only where you want to group tags by category.
        from rapidata import Tag

        benchmark.add_prompts(
            identifiers=["id3"],
            prompts=["prompt 3"],
            tags=[[Tag("landscape", category="scene"), "outdoor"]],
        )
        ```
    """
    with tracer.start_as_current_span("RapidataBenchmark.add_prompts"):
        if prompts and (
            not isinstance(prompts, list)
            or not all(
                isinstance(prompt, str) or prompt is None for prompt in prompts
            )
        ):
            raise ValueError("Prompts must be a list of strings or None.")

        if prompt_assets and (
            not isinstance(prompt_assets, list)
            or not all(
                isinstance(asset, str) or asset is None for asset in prompt_assets
            )
        ):
            raise ValueError("Media assets must be a list of strings or None.")

        if identifiers and (
            not isinstance(identifiers, list)
            or not all(isinstance(identifier, str) for identifier in identifiers)
        ):
            raise ValueError("Identifiers must be a list of strings.")

        if identifiers and len(set(identifiers)) != len(identifiers):
            raise ValueError("Identifiers must be unique.")

        if tags is not None:
            if not is_tag_list(tags):
                raise ValueError("Tags must be a list of lists of str/Tag or None.")

            for tag in tags:
                if tag is not None and (
                    not is_tag_list(tag)
                    or not all(isinstance(item, (str, Tag)) for item in tag)
                ):
                    raise ValueError(
                        "Tags must be a list of lists of str/Tag or None."
                    )

        if origins is not None:
            if not is_tag_list(origins) or not all(
                origin is None or isinstance(origin, (Origin, str))
                for origin in origins
            ):
                raise ValueError("Origins must be a list of Origin/str or None.")

        if not identifiers and not prompts:
            raise ValueError(
                "At least one of identifiers or prompts must be provided."
            )

        if not prompts and not prompt_assets:
            raise ValueError(
                "At least one of prompts or media assets must be provided."
            )

        if not identifiers:
            assert prompts is not None
            if len(set(prompts)) != len(prompts):
                raise ValueError(
                    "Prompts must be unique. Otherwise use identifiers."
                )
            if any(prompt is None for prompt in prompts):
                raise ValueError(
                    "Prompts must not be None. Otherwise use identifiers."
                )

            identifiers = cast(list[str], prompts)

        assert identifiers is not None

        expected_length = len(identifiers)

        if not prompts:
            prompts = cast(list[str | None], [None] * expected_length)

        if not prompt_assets:
            prompt_assets = cast(list[str | None], [None] * expected_length)

        if not tags:
            tags = cast(
                Sequence[Sequence[str | Tag] | None], [None] * expected_length
            )

        if not origins:
            origins = cast(Sequence[Origin | str | None], [None] * expected_length)

        if not (
            expected_length
            == len(prompts)
            == len(prompt_assets)
            == len(tags)
            == len(origins)
        ):
            raise ValueError(
                "Identifiers, prompts, media assets, tags, and origins must have the same length or set to None."
            )

        # Snapshot once: `self.identifiers` is a property whose getter re-fetches
        # over HTTP while the cache is empty, so testing it inside the comprehension
        # fired one request per identifier (a full re-fetch each time on a fresh,
        # empty benchmark). One lookup into a set instead.
        existing_identifiers = set(self.identifiers)
        already_registered = [
            identifier
            for identifier in identifiers
            if identifier in existing_identifiers
        ]
        if already_registered:
            raise ValueError(
                f"Identifiers already exist in the benchmark: {already_registered}"
            )

        to_upload = [
            BenchmarkPrompt(
                identifier,
                prompt,
                asset,
                coerce_tags(tag),
                coerce_origin(origin),
            )
            for identifier, prompt, asset, tag, origin in zip(
                identifiers, prompts, prompt_assets, tags, origins
            )
        ]

        for uploaded in self._prompt_uploader.upload_many(to_upload):
            self.__identifiers.append(uploaded.identifier)
            self.__prompts.append(uploaded.prompt)
            self.__prompt_assets.append(
                self.__normalize_cached_asset(uploaded.prompt_asset)
            )
            self.__structured_tags.append(uploaded.tags)
            self.__tags.append([tag.value for tag in uploaded.tags])
            self.__origins.append(uploaded.origin)

        # The English translation is produced server-side and is unknown for
        # the just-added prompts. Clear it so the next access lazily re-fetches
        # the prompt set, while the rest of the cache stays intact.
        self.__english_prompts = []

update_prompt

update_prompt(
    identifier: str,
    tags: Optional[Sequence[str | Tag]] = None,
    origin: Origin | str | None = None,
) -> None

Updates the tags and/or origin of an existing prompt.

Only the provided fields are changed: pass tags to replace the prompt's tags, origin to set its origin, or both. A field left as None is not sent and the server leaves it unchanged.

Parameters:

Name	Type	Description	Default
identifier	str	The identifier of the prompt to update. Must already be registered on the benchmark.	required
tags	Optional[Sequence[str | Tag]]	The new tags for the prompt — plain strings, :class:Tag objects, or a mix; strings are converted to Tag(value, category=None). Replaces the existing tags.	None
origin	Origin | str | None	The new origin (an :class:Origin or a plain string mapped to Origin(source)).	None

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def update_prompt(
    self,
    identifier: str,
    tags: Optional[Sequence[str | Tag]] = None,
    origin: Origin | str | None = None,
) -> None:
    """Updates the tags and/or origin of an existing prompt.

    Only the provided fields are changed: pass ``tags`` to replace the
    prompt's tags, ``origin`` to set its origin, or both. A field left as
    ``None`` is not sent and the server leaves it unchanged.

    Args:
        identifier: The identifier of the prompt to update. Must already be registered on the benchmark.
        tags: The new tags for the prompt — plain strings, :class:`Tag` objects, or a mix; strings are converted to `Tag(value, category=None)`. Replaces the existing tags.
        origin: The new origin (an :class:`Origin` or a plain string mapped to `Origin(source)`).
    """
    from rapidata.api_client.models.update_prompt_tags_endpoint_input import (
        UpdatePromptTagsEndpointInput,
    )
    from rapidata.api_client.models.tag import Tag as ApiTag
    from rapidata.api_client.models.origin import Origin as ApiOrigin

    with tracer.start_as_current_span("RapidataBenchmark.update_prompt"):
        if tags is None and origin is None:
            raise ValueError("Provide tags and/or origin to update.")

        if tags is not None and (
            not is_tag_list(tags)
            or not all(isinstance(item, (str, Tag)) for item in tags)
        ):
            raise ValueError("Tags must be a list of str/Tag.")

        if origin is not None and not isinstance(origin, (Origin, str)):
            raise ValueError("Origin must be an Origin or str.")

        if identifier not in self.__prompt_ids:
            self.__instantiate_prompts()
        if identifier not in self.__prompt_ids:
            raise ValueError(
                f"Identifier does not exist in the benchmark: {identifier}"
            )
        prompt_id = self.__prompt_ids[identifier]

        structured_tags = coerce_tags(tags) if tags is not None else None
        resolved_origin = coerce_origin(origin)

        self._openapi_service.leaderboard.prompt_api.benchmark_prompt_prompt_id_tags_put(
            prompt_id=prompt_id,
            update_prompt_tags_endpoint_input=UpdatePromptTagsEndpointInput(
                tags=(
                    [
                        ApiTag(value=tag.value, category=tag.category)
                        for tag in structured_tags
                    ]
                    if structured_tags is not None
                    else None
                ),
                origin=(
                    ApiOrigin(source=resolved_origin.source)
                    if resolved_origin is not None
                    else None
                ),
            ),
        )

        # Reflect the change in the caches so a subsequent read is consistent
        # without a re-fetch.
        if identifier in self.__identifiers:
            index = self.__identifiers.index(identifier)
            if structured_tags is not None:
                self.__structured_tags[index] = structured_tags
                self.__tags[index] = [tag.value for tag in structured_tags]
            if resolved_origin is not None:
                self.__origins[index] = resolved_origin

create_leaderboard

create_leaderboard(
    name: str,
    instruction: str,
    show_prompt: bool = False,
    show_prompt_asset: bool = False,
    inverse_ranking: bool = False,
    level_of_detail: LevelOfDetail | int | None = None,
    min_responses_per_matchup: int | None = None,
    audience_id: str | RapidataAudienceBase | None = None,
    settings: Sequence["RapidataSetting"] | None = None,
    included_tags: list[str] | None = None,
    excluded_tags: list[str] | None = None,
) -> RapidataLeaderboard

Creates a new leaderboard for the benchmark.

Parameters:

Name	Type	Description	Default
name	str	The name of the leaderboard. (not shown to the users)	required
instruction	str	The instruction decides how the models will be evaluated.	required
show_prompt	bool	Whether to show the prompt to the users. (default: False)	False
show_prompt_asset	bool	Whether to show the prompt asset to the users. (only works if the prompt asset is a URL) (default: False)	False
inverse_ranking	bool	Whether to inverse the ranking of the leaderboard. (if the question is inversed, e.g. "Which video is worse?")	False
level_of_detail	LevelOfDetail | int | None	Sets the leaderboard's response budget — the total number of comparison responses collected per model evaluation. A larger budget buys more matchups and therefore more precise standings, at the cost of a slower, more expensive evaluation. Either one of the named levels — 'debug' (20 responses), 'low' (2,000), 'medium' (4,000), 'high' (8,000), 'very high' (16,000) — or a positive integer for a custom budget. (default: None, server decides)	None
min_responses_per_matchup	int | None	The minimum number of responses required to be considered for the leaderboard. (default: 3)	None
audience_id	str | RapidataAudienceBase | None	The audience that should answer the leaderboard. Pass either the audience id, a :class:RapidataAudience (dimension audience), or a :class:RapidataFilteredAudience (derived via meth:RapidataAudience.filter). Defaults to the global audience when not specified.	None
settings	Sequence['RapidataSetting'] | None	The settings that should be applied to the leaderboard. Will determine the behavior of the tasks on the leaderboard. (default: [])	None
included_tags	list[str] | None	Restricts which of the benchmark's prompts this leaderboard collects matchups for: only prompts carrying at least one of these tag values are used. When empty or not specified (the default) every prompt is eligible. Note that a non-empty list drops untagged prompts. (default: None)	None
excluded_tags	list[str] | None	Prompt tag values to skip when collecting matchups. Always wins over included_tags — a prompt carrying both an included and an excluded tag is skipped. (default: None)	None

Do not confuse either tag argument with the tags argument of :meth:RapidataLeaderboard.get_standings: that filters the standings you read back out, whereas these decide which prompts get matchups collected in the first place.

Both match on the tag value; a tag's category is irrelevant. They are resolved when a run starts rather than snapshotted at creation, so re-tagging a prompt changes which future runs pick it up. They are read back via the read-only :attr:RapidataLeaderboard.included_tags / :attr:RapidataLeaderboard.excluded_tags and cannot be changed afterwards — to re-scope, create a new leaderboard.

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def create_leaderboard(
    self,
    name: str,
    instruction: str,
    show_prompt: bool = False,
    show_prompt_asset: bool = False,
    inverse_ranking: bool = False,
    level_of_detail: LevelOfDetail | int | None = None,
    min_responses_per_matchup: int | None = None,
    audience_id: str | RapidataAudienceBase | None = None,
    settings: Sequence["RapidataSetting"] | None = None,
    included_tags: list[str] | None = None,
    excluded_tags: list[str] | None = None,
) -> RapidataLeaderboard:
    """
    Creates a new leaderboard for the benchmark.

    Args:
        name: The name of the leaderboard. (not shown to the users)
        instruction: The instruction decides how the models will be evaluated.
        show_prompt: Whether to show the prompt to the users. (default: False)
        show_prompt_asset: Whether to show the prompt asset to the users. (only works if the prompt asset is a URL) (default: False)
        inverse_ranking: Whether to inverse the ranking of the leaderboard. (if the question is inversed, e.g. "Which video is worse?")
        level_of_detail: Sets the leaderboard's response budget — the total number of comparison responses collected per model evaluation. A larger budget buys more matchups and therefore more precise standings, at the cost of a slower, more expensive evaluation. Either one of the named levels — 'debug' (20 responses), 'low' (2,000), 'medium' (4,000), 'high' (8,000), 'very high' (16,000) — or a positive integer for a custom budget. (default: None, server decides)
        min_responses_per_matchup: The minimum number of responses required to be considered for the leaderboard. (default: 3)
        audience_id: The audience that should answer the leaderboard. Pass either the audience id, a :class:`RapidataAudience` (dimension audience), or a :class:`RapidataFilteredAudience` (derived via :py:meth:`RapidataAudience.filter`). Defaults to the global audience when not specified.
        settings: The settings that should be applied to the leaderboard. Will determine the behavior of the tasks on the leaderboard. (default: [])
        included_tags: Restricts **which of the benchmark's prompts this leaderboard collects matchups for**: only prompts carrying at least one of these tag values are used. When empty or not specified (the default) every prompt is eligible. Note that a non-empty list drops untagged prompts. (default: None)
        excluded_tags: Prompt tag values to skip when collecting matchups. Always wins over ``included_tags`` — a prompt carrying both an included and an excluded tag is skipped. (default: None)

    Do not confuse either tag argument with the ``tags`` argument of
    :meth:`RapidataLeaderboard.get_standings`: that filters the standings you read
    back out, whereas these decide which prompts get matchups collected in the first
    place.

    Both match on the tag **value**; a tag's category is irrelevant. They are
    resolved when a run starts rather than snapshotted at creation, so re-tagging a
    prompt changes which future runs pick it up. They are read back via the
    read-only :attr:`RapidataLeaderboard.included_tags` /
    :attr:`RapidataLeaderboard.excluded_tags` and cannot be changed afterwards — to
    re-scope, create a new leaderboard.
    """
    from rapidata.api_client.models.create_leaderboard_endpoint_input import (
        CreateLeaderboardEndpointInput,
    )
    from rapidata.rapidata_client.audience._audience_base import (
        RapidataAudienceBase,
    )
    from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
    from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import (
        RapidataLeaderboard,
    )

    with tracer.start_as_current_span("RapidataBenchmark.create_leaderboard"):
        response_budget = (
            DetailMapper.resolve_budget(level_of_detail)
            if level_of_detail is not None
            else None
        )

        if min_responses_per_matchup is not None and (
            not isinstance(min_responses_per_matchup, int)
            or min_responses_per_matchup < 3
        ):
            raise ValueError(
                "Min responses per matchup must be an integer and at least 3"
            )

        resolved_audience_id = (
            audience_id.id
            if isinstance(audience_id, RapidataAudienceBase)
            else audience_id
        )

        logger.info(
            "Creating leaderboard %s with instruction %s, show_prompt %s, show_prompt_asset %s, inverse_ranking %s, level_of_detail %s, min_responses_per_matchup %s, audience_id %s, settings %s, included_tags %s, excluded_tags %s",
            name,
            instruction,
            show_prompt,
            show_prompt_asset,
            inverse_ranking,
            level_of_detail,
            min_responses_per_matchup,
            resolved_audience_id,
            settings,
            included_tags,
            excluded_tags,
        )

        leaderboard_result = (
            self._openapi_service.leaderboard.leaderboard_api.leaderboard_post(
                create_leaderboard_endpoint_input=CreateLeaderboardEndpointInput(
                    benchmarkId=self.id,
                    name=name,
                    instruction=instruction,
                    showPrompt=show_prompt,
                    showPromptAsset=show_prompt_asset,
                    isInversed=inverse_ranking,
                    minResponses=min_responses_per_matchup,
                    responseBudget=response_budget,
                    audienceId=resolved_audience_id,
                    includedTags=included_tags,
                    excludedTags=excluded_tags,
                    featureFlags=(
                        [setting._to_feature_flag() for setting in settings]
                        if settings
                        else None
                    ),
                )
            )
        )

        assert (
            leaderboard_result.benchmark_id == self.id
        ), "The leaderboard was not created for the correct benchmark."

        logger.info("Leaderboard created with id %s", leaderboard_result.id)

        return RapidataLeaderboard(
            name,
            instruction,
            show_prompt,
            show_prompt_asset,
            inverse_ranking,
            leaderboard_result.response_budget,
            leaderboard_result.min_responses,
            self.id,
            leaderboard_result.id,
            self._openapi_service,
            included_tags,
            excluded_tags,
        )

evaluate_model

evaluate_model(
    name: str,
    media: list[str],
    identifiers: list[str] | None = None,
    prompts: list[str] | None = None,
    data_type: Literal["media", "text"] = "media",
) -> None

Evaluates a model on the benchmark across all leaderboards.

prompts or identifiers must be provided to match the media.

Parameters:

Name	Type	Description	Default
name	str	The name of the model.	required
media	list[str]	The generated media or text that will be used to evaluate the model.	required
identifiers	list[str] | None	The identifiers that correspond to the media. The order of the identifiers must match the order of the media. The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property.	None
prompts	list[str] | None	The prompts that correspond to the media. The order of the prompts must match the order of the media.	None
data_type	Literal['media', 'text']	The type of data being provided. Use "media" for images/videos/audio (default) or "text" for text content.	'media'

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def evaluate_model(
    self,
    name: str,
    media: list[str],
    identifiers: list[str] | None = None,
    prompts: list[str] | None = None,
    data_type: Literal["media", "text"] = "media",
) -> None:
    """
    Evaluates a model on the benchmark across all leaderboards.

    prompts or identifiers must be provided to match the media.

    Args:
        name: The name of the model.
        media: The generated media or text that will be used to evaluate the model.
        identifiers: The identifiers that correspond to the media. The order of the identifiers must match the order of the media.\n
            The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property.
        prompts: The prompts that correspond to the media. The order of the prompts must match the order of the media.
        data_type: The type of data being provided. Use "media" for images/videos/audio (default) or "text" for text content.
    """
    with tracer.start_as_current_span("RapidataBenchmark.evaluate_model"):
        participant = self.add_model(
            name=name,
            media=media,
            identifiers=identifiers,
            prompts=prompts,
            data_type=data_type,
        )
        participant.run()

add_model

add_model(
    name: str,
    media: list[str],
    identifiers: list[str] | None = None,
    prompts: list[str] | None = None,
    data_type: Literal["media", "text"] = "media",
) -> BenchmarkParticipant

Adds a model to the benchmark without immediately submitting it for evaluation.

This method creates a participant, uploads media, but does NOT submit the participant. Use participant.run() or benchmark.run() to submit afterwards.

Parameters:

Name	Type	Description	Default
name	str	The name of the model.	required
media	list[str]	The generated media or text that will be used to evaluate the model.	required
identifiers	list[str] | None	The identifiers that correspond to the media. The order of the identifiers must match the order of the media. The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property.	None
prompts	list[str] | None	The prompts that correspond to the media. The order of the prompts must match the order of the media.	None
data_type	Literal['media', 'text']	The type of data being provided. Use "media" for images/videos/audio (default) or "text" for text content.	'media'

Returns:

Type	Description
BenchmarkParticipant	The created BenchmarkParticipant instance.

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def add_model(
    self,
    name: str,
    media: list[str],
    identifiers: list[str] | None = None,
    prompts: list[str] | None = None,
    data_type: Literal["media", "text"] = "media",
) -> BenchmarkParticipant:
    """Adds a model to the benchmark without immediately submitting it for evaluation.

    This method creates a participant, uploads media, but does NOT submit the participant.
    Use `participant.run()` or `benchmark.run()` to submit afterwards.

    Args:
        name: The name of the model.
        media: The generated media or text that will be used to evaluate the model.
        identifiers: The identifiers that correspond to the media. The order of the identifiers must match the order of the media.\n
            The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property.
        prompts: The prompts that correspond to the media. The order of the prompts must match the order of the media.
        data_type: The type of data being provided. Use "media" for images/videos/audio (default) or "text" for text content.

    Returns:
        The created BenchmarkParticipant instance.
    """
    from rapidata.api_client.models.create_benchmark_participant_endpoint_input import (
        CreateBenchmarkParticipantEndpointInput,
    )
    from rapidata.rapidata_client.benchmark.participant.participant import (
        BenchmarkParticipant,
    )

    with tracer.start_as_current_span("RapidataBenchmark.add_model"):
        if not media:
            raise ValueError("Media must be a non-empty list of strings")

        if not identifiers and not prompts:
            raise ValueError("Identifiers or prompts must be provided.")

        if identifiers and prompts:
            raise ValueError(
                "Identifiers and prompts cannot be provided at the same time. Use one or the other."
            )

        if not identifiers:
            assert prompts is not None
            identifiers = prompts

        if len(media) != len(identifiers):
            raise ValueError(
                "Media and identifiers/prompts must have the same length"
            )

        if not all(identifier in self.identifiers for identifier in identifiers):
            raise ValueError(
                "All identifiers/prompts must be in the registered identifiers/prompts list. To see the registered identifiers/prompts, use the identifiers/prompts property."
            )

        participant_result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_participants_post(
            benchmark_id=self.id,
            create_benchmark_participant_endpoint_input=CreateBenchmarkParticipantEndpointInput(
                name=name,
            ),
        )

        logger.info(f"Participant created: {participant_result.participant_id}")

        participant = BenchmarkParticipant(
            name,
            participant_result.participant_id,
            self._openapi_service,
            self.id,
        )

        with tracer.start_as_current_span("upload_media_for_participant"):
            logger.info(
                f"Uploading {len(media)} media assets to participant {participant.id}"
            )

            successful_uploads, failed_uploads = participant.upload_media(
                media,
                identifiers,
                data_type=data_type,
            )

            total_uploads = len(media)
            success_rate = (
                (len(successful_uploads) / total_uploads * 100)
                if total_uploads > 0
                else 0
            )
            logger.info(
                f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)"
            )

            if failed_uploads:
                logger.error(f"Failed uploads for media: {failed_uploads}")
                logger.warning(
                    "Some uploads failed. The model evaluation may be incomplete."
                )

            if len(successful_uploads) == 0:
                raise RuntimeError(
                    "No uploads were successful. The model evaluation will not be completed."
                )

        # Clear cache so next access re-fetches
        self.__participants = []

        return participant

run

run() -> None

Submits all participants that are in CREATED state.

This is a convenience method to submit all unsubmitted participants at once.

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def run(self) -> None:
    """Submits all participants that are in `CREATED` state.

    This is a convenience method to submit all unsubmitted participants at once.
    """
    from rapidata.api_client.models.participant_status import ParticipantStatus

    with tracer.start_as_current_span("RapidataBenchmark.run"):
        created = [
            p for p in self.participants if p.status == ParticipantStatus.CREATED
        ]
        logger.info(f"Submitting {len(created)} participants in CREATED state")
        for participant in created:
            participant.run()

        # Clear cache so next access re-fetches
        self.__participants = []

view

view() -> None

Views the benchmark.

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def view(self) -> None:
    """
    Views the benchmark.
    """

    logger.info("Opening benchmark page in browser...")
    could_open_browser = webbrowser.open(self.__benchmark_page)
    if not could_open_browser:
        encoded_url = urllib.parse.quote(
            self.__benchmark_page, safe="%/:=&?~#+!$,;'@()*[]"
        )
        managed_print(
            Fore.RED
            + f"Please open this URL in your browser: '{encoded_url}'"
            + Fore.RESET
        )

get_overall_standings

get_overall_standings(
    tags: Optional[list[str]] = None,
    leaderboard_ids: Optional[list[str]] = None,
) -> DataFrame

Returns an aggregated elo table of all leaderboards in the benchmark.

Parameters:

Name	Type	Description	Default
tags	Optional[list[str]]	Filter standings by these tags. If None, all tags are considered.	None
leaderboard_ids	Optional[list[str]]	Filter to only include matchups from these leaderboards. If None, all leaderboards are considered.	None

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def get_overall_standings(
    self,
    tags: Optional[list[str]] = None,
    leaderboard_ids: Optional[list[str]] = None,
) -> pd.DataFrame:
    """
    Returns an aggregated elo table of all leaderboards in the benchmark.

    Args:
        tags: Filter standings by these tags. If None, all tags are considered.
        leaderboard_ids: Filter to only include matchups from these leaderboards. If None, all leaderboards are considered.
    """
    import pandas as pd

    with tracer.start_as_current_span("get_overall_standings"):
        tags_filter = AudienceAudienceIdJobsGetJobIdParameter()
        tags_filter.var_in = tags
        leaderboard_filter = AudienceAudienceIdJobsGetJobIdParameter()
        leaderboard_filter.var_in = leaderboard_ids

        participants = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_standings_query_get(
            benchmark_id=self.id,
            tags=tags_filter,
            leaderboard_id=leaderboard_filter,
        )

        standings = []
        for participant in participants.items:
            standings.append(
                {
                    "name": participant.name,
                    "wins": participant.wins,
                    "total_matches": participant.total_matches,
                    "score": (
                        round(participant.score, 2)
                        if participant.score is not None
                        else None
                    ),
                }
            )

        return pd.DataFrame(standings)

get_win_loss_matrix

get_win_loss_matrix(
    tags: Optional[list[str]] = None,
    participant_ids: Optional[list[str]] = None,
    leaderboard_ids: Optional[list[str]] = None,
    use_weighted_scoring: Optional[bool] = None,
) -> DataFrame

Returns the pairwise win/loss matrix aggregated across the benchmark's leaderboards.

The returned DataFrame is square, with participant names on both the index (rows) and columns. Cell [i, j] is how often participant i (row) beat participant j (column) in their direct matchups, summed over every leaderboard in scope. Read a row to see how a model did against every opponent; the diagonal (a model against itself) is always 0. This is the head-to-head breakdown behind :meth:get_overall_standings, which collapses the same matchups into a single Elo score per model.

Parameters:

Name	Type	Description	Default
tags	Optional[list[str]]	Only count matchups carrying one of these prompt tags. If None, every matchup is included; if an empty list, none are.	None
participant_ids	Optional[list[str]]	Restrict the matrix to these participants. If None, all participants are included.	None
leaderboard_ids	Optional[list[str]]	Only aggregate matchups from these leaderboards. If None, all leaderboards in the benchmark are included.	None
use_weighted_scoring	Optional[bool]	If True, each matchup is weighted by the responding annotators' reliability (userScore) instead of being counted as a plain win, so cells hold weighted sums (floats) rather than raw counts. If False, cells are raw win counts. When None (default), the server applies its configured default.	None

Returns:

Type	Description
DataFrame	A pandas DataFrame indexed by participant name on both axes, where cell
DataFrame	[i, j] holds the (optionally weighted) number of wins of the row
DataFrame	participant over the column participant.

Source code in src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

def get_win_loss_matrix(
    self,
    tags: Optional[list[str]] = None,
    participant_ids: Optional[list[str]] = None,
    leaderboard_ids: Optional[list[str]] = None,
    use_weighted_scoring: Optional[bool] = None,
) -> pd.DataFrame:
    """
    Returns the pairwise win/loss matrix aggregated across the benchmark's leaderboards.

    The returned DataFrame is square, with participant names on both the index
    (rows) and columns. Cell ``[i, j]`` is how often participant ``i`` (row) beat
    participant ``j`` (column) in their direct matchups, summed over every
    leaderboard in scope. Read a row to see how a model did against every
    opponent; the diagonal (a model against itself) is always 0. This is the
    head-to-head breakdown behind :meth:`get_overall_standings`, which collapses
    the same matchups into a single Elo score per model.

    Args:
        tags: Only count matchups carrying one of these prompt tags. If None,
            every matchup is included; if an empty list, none are.
        participant_ids: Restrict the matrix to these participants. If None, all
            participants are included.
        leaderboard_ids: Only aggregate matchups from these leaderboards. If None,
            all leaderboards in the benchmark are included.
        use_weighted_scoring: If True, each matchup is weighted by the responding
            annotators' reliability (``userScore``) instead of being counted as a
            plain win, so cells hold weighted sums (floats) rather than raw counts.
            If False, cells are raw win counts. When None (default), the server
            applies its configured default.

    Returns:
        A pandas DataFrame indexed by participant name on both axes, where cell
        ``[i, j]`` holds the (optionally weighted) number of wins of the row
        participant over the column participant.
    """
    import pandas as pd

    with tracer.start_as_current_span("get_win_loss_matrix"):
        tags_filter = AudienceAudienceIdJobsGetJobIdParameter()
        tags_filter.var_in = tags
        participant_filter = AudienceAudienceIdJobsGetJobIdParameter()
        participant_filter.var_in = participant_ids
        leaderboard_filter = AudienceAudienceIdJobsGetJobIdParameter()
        leaderboard_filter.var_in = leaderboard_ids

        result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_matrix_query_get(
            benchmark_id=self.id,
            tags=tags_filter,
            participant_id=participant_filter,
            leaderboard_id=leaderboard_filter,
            use_weighted_scoring=use_weighted_scoring,
        )

        return pd.DataFrame(
            data=result.data,
            index=pd.Index(result.index),
            columns=pd.Index(result.columns),
        )