Rapidata leaderboard

RapidataLeaderboard #

RapidataLeaderboard(
    name: str,
    instruction: str,
    show_prompt: bool,
    show_prompt_asset: bool,
    inverse_ranking: bool,
    response_budget: int,
    min_responses_per_matchup: int,
    id: str,
    openapi_service: OpenAPIService,
)

An instance of a Rapidata leaderboard.

Used to interact with a specific leaderboard in the Rapidata system, such as retrieving prompts and evaluating models.

Parameters:

Name	Type	Description	Default
`name`	`str`	The name that will be used to identify the leaderboard on the overview.	required
`instruction`	`str`	The instruction that will determine what how the models will be evaluated.	required
`show_prompt`	`bool`	Whether to show the prompt to the users.	required
`id`	`str`	The ID of the leaderboard.	required
`openapi_service`	`OpenAPIService`	The OpenAPIService instance for API interaction.	required

Source code in src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

def __init__(
    self,
    name: str,
    instruction: str,
    show_prompt: bool,
    show_prompt_asset: bool,
    inverse_ranking: bool,
    response_budget: int,
    min_responses_per_matchup: int,
    id: str,
    openapi_service: OpenAPIService,
):
    self.__openapi_service = openapi_service
    self.__name = name
    self.__instruction = instruction
    self.__show_prompt = show_prompt
    self.__show_prompt_asset = show_prompt_asset
    self.__inverse_ranking = inverse_ranking
    self.__response_budget = response_budget
    self.__min_responses_per_matchup = min_responses_per_matchup
    self.id = id

level_of_detail `property` `writable` #

level_of_detail: Literal[
    "low", "medium", "high", "very high"
]

Returns the level of detail of the leaderboard.

min_responses_per_matchup `property` `writable` #

min_responses_per_matchup: int

Returns the minimum number of responses required to be considered for the leaderboard.

show_prompt_asset `property` #

show_prompt_asset: bool

Returns whether the prompt asset is shown to the users.

inverse_ranking `property` #

inverse_ranking: bool

Returns whether the ranking is inverse.

show_prompt `property` #

show_prompt: bool

Returns whether the prompt is shown to the users.

instruction `property` #

instruction: str

Returns the instruction of the leaderboard.

name `property` #

name: str

Returns the name of the leaderboard.

get_standings #

get_standings(
    tags: Optional[list[str]] = None,
) -> DataFrame

Returns the standings of the leaderboard.

Parameters:

Name	Type	Description	Default
`tags`	`Optional[list[str]]`	The matchups with these tags should be used to create the standings. If tags are None, all matchups will be considered. If tags are empty, no matchups will be considered.	`None`

Returns:

Type	Description
`DataFrame`	A pandas DataFrame containing the standings of the leaderboard.

Source code in src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

def get_standings(self, tags: Optional[list[str]] = None) -> pd.DataFrame:
    """
    Returns the standings of the leaderboard.

    Args:
        tags: The matchups with these tags should be used to create the standings.
            If tags are None, all matchups will be considered.
            If tags are empty, no matchups will be considered.

    Returns:
        A pandas DataFrame containing the standings of the leaderboard.
    """

    participants = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_standings_get(
        leaderboard_id=self.id, tags=tags
    )

    standings = []
    for participant in participants.items:
        standings.append(
            {
                "name": participant.name,
                "wins": participant.wins,
                "total_matches": participant.total_matches,
                "score": (
                    round(participant.score, 2)
                    if participant.score is not None
                    else None
                ),
            }
        )

    return pd.DataFrame(standings)

Rapidata leaderboard