tea_tasting.experiment

Experiment and experiment result.

Experiment(metrics=None, variant='variant', **kw_metrics)

Bases: ReprMixin

Experiment definition: metrics and variant column.

Parameters:

Name	Type	Description	Default
metrics	dict[str, MetricBase[Any]] | None	Dictionary of metrics with metric names as keys.	None
variant	str	Variant column name.	'variant'
kw_metrics	MetricBase[Any]	Metrics with metric names as parameter names.	{}

Examples:

>>> import tea_tasting as tt

>>> experiment = tt.Experiment(
...     sessions_per_user=tt.Mean("sessions"),
...     orders_per_session=tt.RatioOfMeans("orders", "sessions"),
...     orders_per_user=tt.Mean("orders"),
...     revenue_per_user=tt.Mean("revenue"),
... )
>>> data = tt.make_users_data(seed=42)
>>> result = experiment.analyze(data)
>>> result
            metric control treatment rel_effect_size rel_effect_size_ci pvalue
 sessions_per_user    2.00      1.98          -0.66%      [-3.7%, 2.5%]  0.674
orders_per_session   0.266     0.289            8.8%      [-0.89%, 19%] 0.0762
   orders_per_user   0.530     0.573            8.0%       [-2.0%, 19%]  0.118
  revenue_per_user    5.24      5.73            9.3%       [-2.4%, 22%]  0.123

Using the first argument metrics which accepts metrics in a form of dictionary:

>>> experiment = tt.Experiment({
...     "sessions per user": tt.Mean("sessions"),
...     "orders per session": tt.RatioOfMeans("orders", "sessions"),
...     "orders per user": tt.Mean("orders"),
...     "revenue per user": tt.Mean("revenue"),
... })
>>> data = tt.make_users_data(seed=42)
>>> result = experiment.analyze(data)
>>> result
            metric control treatment rel_effect_size rel_effect_size_ci pvalue
 sessions per user    2.00      1.98          -0.66%      [-3.7%, 2.5%]  0.674
orders per session   0.266     0.289            8.8%      [-0.89%, 19%] 0.0762
   orders per user   0.530     0.573            8.0%       [-2.0%, 19%]  0.118
  revenue per user    5.24      5.73            9.3%       [-2.4%, 22%]  0.123

Power analysis:

>>> data = tt.make_users_data(
...     seed=42,
...     sessions_uplift=0,
...     orders_uplift=0,
...     revenue_uplift=0,
...     covariates=True,
... )
>>> with tt.config_context(n_obs=(10_000, 20_000)):
...     experiment = tt.Experiment(
...         sessions_per_user=tt.Mean("sessions", "sessions_covariate"),
...         orders_per_session=tt.RatioOfMeans(
...             numer="orders",
...             denom="sessions",
...             numer_covariate="orders_covariate",
...             denom_covariate="sessions_covariate",
...         ),
...         orders_per_user=tt.Mean("orders", "orders_covariate"),
...         revenue_per_user=tt.Mean("revenue", "revenue_covariate"),
...     )
>>> power_result = experiment.solve_power(data)
>>> power_result
            metric power effect_size rel_effect_size n_obs
 sessions_per_user   80%      0.0458            2.3% 10000
 sessions_per_user   80%      0.0324            1.6% 20000
orders_per_session   80%      0.0177            6.8% 10000
orders_per_session   80%      0.0125            4.8% 20000
   orders_per_user   80%      0.0374            7.2% 10000
   orders_per_user   80%      0.0264            5.1% 20000
  revenue_per_user   80%       0.488            9.2% 10000
  revenue_per_user   80%       0.345            6.5% 20000

Source code in src/tea_tasting/experiment.py

def __init__(
    self,
    metrics: dict[str, tea_tasting.metrics.MetricBase[Any]] | None = None,
    variant: str = "variant",
    **kw_metrics: tea_tasting.metrics.MetricBase[Any],
) -> None:
    """Experiment definition: metrics and variant column.

    Args:
        metrics: Dictionary of metrics with metric names as keys.
        variant: Variant column name.
        kw_metrics: Metrics with metric names as parameter names.

    Examples:
        ```pycon
        >>> import tea_tasting as tt

        >>> experiment = tt.Experiment(
        ...     sessions_per_user=tt.Mean("sessions"),
        ...     orders_per_session=tt.RatioOfMeans("orders", "sessions"),
        ...     orders_per_user=tt.Mean("orders"),
        ...     revenue_per_user=tt.Mean("revenue"),
        ... )
        >>> data = tt.make_users_data(seed=42)
        >>> result = experiment.analyze(data)
        >>> result
                    metric control treatment rel_effect_size rel_effect_size_ci pvalue
         sessions_per_user    2.00      1.98          -0.66%      [-3.7%, 2.5%]  0.674
        orders_per_session   0.266     0.289            8.8%      [-0.89%, 19%] 0.0762
           orders_per_user   0.530     0.573            8.0%       [-2.0%, 19%]  0.118
          revenue_per_user    5.24      5.73            9.3%       [-2.4%, 22%]  0.123

        ```

        Using the first argument `metrics` which accepts metrics in a form of dictionary:

        ```pycon
        >>> experiment = tt.Experiment({
        ...     "sessions per user": tt.Mean("sessions"),
        ...     "orders per session": tt.RatioOfMeans("orders", "sessions"),
        ...     "orders per user": tt.Mean("orders"),
        ...     "revenue per user": tt.Mean("revenue"),
        ... })
        >>> data = tt.make_users_data(seed=42)
        >>> result = experiment.analyze(data)
        >>> result
                    metric control treatment rel_effect_size rel_effect_size_ci pvalue
         sessions per user    2.00      1.98          -0.66%      [-3.7%, 2.5%]  0.674
        orders per session   0.266     0.289            8.8%      [-0.89%, 19%] 0.0762
           orders per user   0.530     0.573            8.0%       [-2.0%, 19%]  0.118
          revenue per user    5.24      5.73            9.3%       [-2.4%, 22%]  0.123

        ```

        Power analysis:

        ```pycon
        >>> data = tt.make_users_data(
        ...     seed=42,
        ...     sessions_uplift=0,
        ...     orders_uplift=0,
        ...     revenue_uplift=0,
        ...     covariates=True,
        ... )
        >>> with tt.config_context(n_obs=(10_000, 20_000)):
        ...     experiment = tt.Experiment(
        ...         sessions_per_user=tt.Mean("sessions", "sessions_covariate"),
        ...         orders_per_session=tt.RatioOfMeans(
        ...             numer="orders",
        ...             denom="sessions",
        ...             numer_covariate="orders_covariate",
        ...             denom_covariate="sessions_covariate",
        ...         ),
        ...         orders_per_user=tt.Mean("orders", "orders_covariate"),
        ...         revenue_per_user=tt.Mean("revenue", "revenue_covariate"),
        ...     )
        >>> power_result = experiment.solve_power(data)
        >>> power_result
                    metric power effect_size rel_effect_size n_obs
         sessions_per_user   80%      0.0458            2.3% 10000
         sessions_per_user   80%      0.0324            1.6% 20000
        orders_per_session   80%      0.0177            6.8% 10000
        orders_per_session   80%      0.0125            4.8% 20000
           orders_per_user   80%      0.0374            7.2% 10000
           orders_per_user   80%      0.0264            5.1% 20000
          revenue_per_user   80%       0.488            9.2% 10000
          revenue_per_user   80%       0.345            6.5% 20000

        ```
    """  # noqa: E501
    if metrics is None:
        metrics = {}
    metrics = metrics | kw_metrics

    tea_tasting.utils.check_scalar(metrics, "metrics", typ=dict)
    tea_tasting.utils.check_scalar(len(metrics), "len(metrics)", gt=0)
    for name, metric in metrics.items():
        tea_tasting.utils.check_scalar(name, "metric_name", typ=str)
        tea_tasting.utils.check_scalar(
            metric, name, typ=tea_tasting.metrics.MetricBase)

    self.metrics = metrics
    self.variant = tea_tasting.utils.check_scalar(
        variant, "variant", typ=str)

analyze(data, control=None, *, all_variants=False)

Analyze the experiment.

Parameters:

Name	Type	Description	Default
data	IntoFrame | Table	Experimental data.	required
control	object	Control variant. If None, the variant with the minimal ID is used as a control.	None
all_variants	bool	If True, analyze all pairs of variants. Otherwise, analyze only one pair of variants.	False

Returns:

Type	Description
ExperimentResult | ExperimentResults	Experiment result.

Source code in src/tea_tasting/experiment.py

def analyze(
    self,
    data: narwhals.typing.IntoFrame | ibis.expr.types.Table,
    control: object = None,
    *,
    all_variants: bool = False,
) -> ExperimentResult | ExperimentResults:
    """Analyze the experiment.

    Args:
        data: Experimental data.
        control: Control variant. If `None`, the variant with the minimal ID
            is used as a control.
        all_variants: If `True`, analyze all pairs of variants. Otherwise,
            analyze only one pair of variants.

    Returns:
        Experiment result.
    """
    tea_tasting.utils.check_scalar(all_variants, "all_variants", typ=bool)
    aggregated_data, granular_data = self._read_data(data)

    if aggregated_data is not None:
        variants = aggregated_data.keys()
    elif granular_data is not None:
        variants = granular_data.keys()
    else:
        variants = self._read_variants(data)
    variants = sorted(variants)  # type: ignore

    if control is not None:
        variant_pairs = tuple(
            (control, treatment)
            for treatment in variants
            if treatment != control
        )
    else:
        variant_pairs = tuple(
            (control, treatment)
            for control in variants
            for treatment in variants
            if control < treatment
        )

    if len(variant_pairs) != 1 and not all_variants:
        raise ValueError(
            "all_variants is False, but there are more than one pair of variants.")

    results = ExperimentResults()
    for contr, treat in variant_pairs:
        result = ExperimentResult()
        for name, metric in self.metrics.items():
            result |= {name: self._analyze_metric(
                metric=metric,
                data=data,
                aggregated_data=aggregated_data,
                granular_data=granular_data,
                control=contr,
                treatment=treat,
            )}

        if not all_variants:
            return result

        results |= {(contr, treat): result}

    return results

simulate(data, n_simulations=10000, *, seed=None, ratio=1, treat=None, map_=map, progress=None)

Simulate the experiment analysis multiple times.

Parameters:

Name	Type	Description	Default
data	IntoFrame | Table | DataGenerator	Experimental data or a callable that generates the data.	required
n_simulations	int	Number of simulations.	10000
seed	int | Generator | SeedSequence | None	Random seed.	None
ratio	float | int	Ratio of the number of users in treatment relative to control.	1
treat	Callable[[Table], Table] | None	Treatment function that takes a PyArrow Table as an input and returns an updated PyArrow Table.	None
map_	MapLike[Any]	Map-like function to run simulations.	map
progress	ProgressFn[Any] | type[Iterable[Any]] | None	tqdm-like class or function to show the progress of simulations.	None

Returns:

Type	Description
SimulationResults	Simulation results.

Source code in src/tea_tasting/experiment.py

def simulate(
    self,
    data: narwhals.typing.IntoFrame | ibis.expr.types.Table | DataGenerator,  # type: ignore
    n_simulations: int = 10_000,
    *,
    seed: int | np.random.Generator | np.random.SeedSequence | None = None,
    ratio: float | int = 1,
    treat: Callable[[pa.Table], pa.Table] | None = None,
    map_: MapLike[Any] = map,
    progress: ProgressFn[Any] | type[Iterable[Any]] | None = None,
) -> SimulationResults:
    """Simulate the experiment analysis multiple times.

    Args:
        data: Experimental data or a callable that generates the data.
        n_simulations: Number of simulations.
        seed: Random seed.
        ratio: Ratio of the number of users in treatment relative to control.
        treat: Treatment function that takes a PyArrow Table as an input
            and returns an updated PyArrow Table.
        map_: Map-like function to run simulations.
        progress: tqdm-like class or function to show the progress of simulations.

    Returns:
        Simulation results.
    """
    tea_tasting.utils.check_scalar(n_simulations, "n_simulations", typ=int, gt=0)
    tea_tasting.utils.auto_check(ratio, "ratio")

    if not callable(data):
        gran_cols: set[str] = set()
        for metric in self.metrics.values():
            if isinstance(metric, tea_tasting.metrics.MetricBaseAggregated):
                aggr_cols = metric.aggr_cols
                gran_cols |= (
                    set(aggr_cols.mean_cols) |
                    set(aggr_cols.var_cols) |
                    set(itertools.chain.from_iterable(aggr_cols.cov_cols))
                )
            elif isinstance(metric, tea_tasting.metrics.MetricBaseGranular):
                gran_cols |= set(metric.cols)
            else:
                gran_cols = set()
                break
        cols = tuple(gran_cols)
        data: pa.Table = tea_tasting.metrics.read_granular(data, cols)
        if self.variant in data.column_names:
            data = data.drop_columns(self.variant)

    sim = functools.partial(
        _simulate_once,
        experiment=self,
        data=data,
        ratio=ratio,
        treat=treat,
    )

    results = map_(sim, np.random.default_rng(seed).spawn(n_simulations))
    if progress is not None:
        try:
            results = progress(results, total=n_simulations)  # type: ignore
        except TypeError:
            results = progress(results)  # type: ignore
    return SimulationResults(results)

solve_power(data, parameter='rel_effect_size')

Solve for a parameter of the power of a test.

Parameters:

Name	Type	Description	Default
data	IntoFrame | Table	Sample data.	required
parameter	Literal['power', 'effect_size', 'rel_effect_size', 'n_obs']	Parameter name.	'rel_effect_size'

Returns:

Type	Description
ExperimentPowerResult	Power analysis result.

Source code in src/tea_tasting/experiment.py

def solve_power(
    self,
    data: narwhals.typing.IntoFrame | ibis.expr.types.Table,
    parameter: Literal[
        "power", "effect_size", "rel_effect_size", "n_obs"] = "rel_effect_size",
) -> ExperimentPowerResult:
    """Solve for a parameter of the power of a test.

    Args:
        data: Sample data.
        parameter: Parameter name.

    Returns:
        Power analysis result.
    """
    tea_tasting.utils.check_scalar(
        parameter,
        "parameter",
        in_={"power", "effect_size", "rel_effect_size", "n_obs"},
    )
    aggr_cols = tea_tasting.metrics.AggrCols()
    for metric in self.metrics.values():
        if isinstance(metric, tea_tasting.metrics.PowerBaseAggregated):
            aggr_cols |= metric.aggr_cols

    aggr_data = tea_tasting.aggr.read_aggregates(
        data,
        group_col=None,
        **aggr_cols._asdict(),
    ) if len(aggr_cols) > 0 else tea_tasting.aggr.Aggregates()

    result = ExperimentPowerResult()
    for name, metric in self.metrics.items():
        if isinstance(metric, tea_tasting.metrics.PowerBaseAggregated):
            result |= {name: metric.solve_power(aggr_data, parameter=parameter)}
        elif isinstance(metric, tea_tasting.metrics.PowerBase):
            result |= {name: metric.solve_power(data, parameter=parameter)}

    return result

ExperimentPowerResult

Bases: DictsReprMixin, UserDict[str, MetricPowerResults[Any]]

Result of the analysis of power in a experiment.

to_arrow()

Convert the object to a PyArrow Table.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_arrow(self) -> pa.Table:
    """Convert the object to a PyArrow Table."""
    return pa.Table.from_pylist(self.to_dicts())

to_dicts()

Convert the result to a sequence of dictionaries.

Source code in src/tea_tasting/experiment.py

@tea_tasting.utils._cache_method
def to_dicts(self) -> tuple[dict[str, object], ...]:
    """Convert the result to a sequence of dictionaries."""
    dicts = ()
    for metric, results in self.items():
        dicts = (*dicts, *({"metric": metric} | d for d in results.to_dicts()))
    return dicts

to_html(keys=None, formatter=get_and_format_num, *, max_rows=None, indent=None)

Convert the object to HTML.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None
indent	str | None	Whitespace to insert for each indentation level. If None, do not indent.	None

Returns:

Type	Description
str	A table with results rendered as HTML.

Source code in src/tea_tasting/utils.py

def to_html(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
    indent: str | None = None,
) -> str:
    """Convert the object to HTML.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.
        indent: Whitespace to insert for each indentation level. If `None`,
            do not indent.

    Returns:
        A table with results rendered as HTML.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    table = ET.Element(
        "table",
        {"class": "dataframe", "style": "text-align: right;"},
    )
    thead = ET.SubElement(table, "thead")
    thead_tr = ET.SubElement(thead, "tr")
    for key in keys:
        th = ET.SubElement(thead_tr, "th")
        th.text = key
    tbody = ET.SubElement(table, "tbody")
    for pretty_dict in self.to_pretty_dicts(keys, formatter, max_rows=max_rows):
        tr = ET.SubElement(tbody, "tr")
        for key in keys:
            td = ET.SubElement(tr, "td")
            td.text = pretty_dict[key]
    if indent is not None:
        ET.indent(table, space=indent)
    return ET.tostring(table, encoding="unicode", method="html")

to_pandas()

Convert the object to a Pandas DataFrame.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_pandas(self) -> pd.DataFrame:
    """Convert the object to a Pandas DataFrame."""
    import pandas as pd
    return pd.DataFrame.from_records(self.to_dicts())

to_polars()

Convert the object to a Polars DataFrame.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_polars(self) -> pl.DataFrame:
    """Convert the object to a Polars DataFrame."""
    import polars as pl
    return pl.from_dicts(self.to_dicts())

to_pretty_dicts(keys=None, formatter=get_and_format_num, *, max_rows=None)

Convert the object to a list of dictionaries with formatted values.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None

Returns:

Type	Description
list[dict[str, str]]	List of dictionaries with formatted values.

Source code in src/tea_tasting/utils.py

def to_pretty_dicts(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
) -> list[dict[str, str]]:
    """Convert the object to a list of dictionaries with formatted values.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.

    Returns:
        List of dictionaries with formatted values.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    dicts = self.to_dicts()
    if max_rows <= 0 or len(dicts) <= max_rows:
        return [{key: formatter(data, key) for key in keys} for data in dicts]

    bottom = max_rows // 2
    top = max_rows - bottom
    return (
        [{key: formatter(data, key) for key in keys} for data in dicts[:top]] +
        [dict.fromkeys(keys, "…")] +
        [{key: formatter(data, key) for key in keys} for data in dicts[-bottom:]]
    )

to_string(keys=None, formatter=get_and_format_num, *, max_rows=None)

Convert the object to a string.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None

Returns:

Type	Description
str	A table with results rendered as string.

Source code in src/tea_tasting/utils.py

def to_string(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
) -> str:
    """Convert the object to a string.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.

    Returns:
        A table with results rendered as string.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    pretty_dicts = self.to_pretty_dicts(keys, formatter, max_rows=max_rows)
    widths = {key: len(key) for key in keys}
    for pretty_dict in pretty_dicts:
        for key in keys:
            widths[key] = max(widths[key], len(pretty_dict[key]))

    sep = " "
    rows = [sep.join(key.rjust(widths[key]) for key in keys)]
    rows.extend(
        sep.join(pretty_dict[key].rjust(widths[key]) for key in keys)
        for pretty_dict in pretty_dicts
    )
    return "\n".join(rows)

with_defaults(*, keys=None, max_rows=None)

Copies the object and sets the new default parameters.

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	New default keys for the methods to_pretty_dicts, to_string, and to_html.	None
max_rows	int | None	New default max_rows for the methods to_pretty_dicts, to_string, and to_html.	None

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default keys.

Source code in src/tea_tasting/utils.py

def with_defaults(
    self: DictsReprMixinT,
    *,
    keys: Sequence[str] | None = None,
    max_rows: int | None = None,
) -> DictsReprMixinT:
    """Copies the object and sets the new default parameters.

    Args:
        keys: New default `keys` for the methods `to_pretty_dicts`, `to_string`,
            and `to_html`.
        max_rows: New default `max_rows` for the methods `to_pretty_dicts`,
            `to_string`, and `to_html`.

    Returns:
        A copy of the object with the new default keys.
    """
    new_instance = self.__class__.__new__(self.__class__)
    new_instance.__dict__.update(self.__dict__)
    new_instance._cache = None
    if keys is not None:
        new_instance.default_keys = keys
    if max_rows is not None:
        new_instance.default_max_rows = max_rows
    return new_instance

with_keys(keys)

Copies the object and sets the new default keys.

Parameters:

Name	Type	Description	Default
keys	Sequence[str]	New default keys for the methods to_pretty_dicts, to_string, and to_html.	required

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default keys.

Source code in src/tea_tasting/utils.py

def with_keys(self: DictsReprMixinT, keys: Sequence[str]) -> DictsReprMixinT:
    """Copies the object and sets the new default `keys`.

    Args:
        keys: New default `keys` for the methods `to_pretty_dicts`, `to_string`,
            and `to_html`.

    Returns:
        A copy of the object with the new default `keys`.
    """
    return self.with_defaults(keys=keys)

with_max_rows(max_rows)

Copies the object and sets the new default max_rows.

Parameters:

Name	Type	Description	Default
max_rows	int	New default max_rows for the methods to_pretty_dicts, to_string, and to_html.	required

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default max_rows.

Source code in src/tea_tasting/utils.py

def with_max_rows(self: DictsReprMixinT, max_rows: int) -> DictsReprMixinT:
    """Copies the object and sets the new default `max_rows`.

    Args:
        max_rows: New default `max_rows` for the methods `to_pretty_dicts`,
            `to_string`, and `to_html`.

    Returns:
        A copy of the object with the new default `max_rows`.
    """
    return self.with_defaults(max_rows=max_rows)

ExperimentResult

Bases: DictsReprMixin, UserDict[str, MetricResult]

Experiment result for a pair of variants.

to_arrow()

Convert the object to a PyArrow Table.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_arrow(self) -> pa.Table:
    """Convert the object to a PyArrow Table."""
    return pa.Table.from_pylist(self.to_dicts())

to_dicts()

Convert the result to a sequence of dictionaries.

Examples:

>>> import pprint
>>> import tea_tasting as tt

>>> experiment = tt.Experiment(
...     orders_per_user=tt.Mean("orders"),
...     revenue_per_user=tt.Mean("revenue"),
... )
>>> data = tt.make_users_data(seed=42)
>>> result = experiment.analyze(data)
>>> pprint.pprint(result.to_dicts())
({'control': 0.5304003954522986,
  'effect_size': 0.04269014577177832,
  'effect_size_ci_lower': -0.010800201598205515,
  'effect_size_ci_upper': 0.09618049314176216,
  'metric': 'orders_per_user',
  'pvalue': np.float64(0.11773177998716214),
  'rel_effect_size': 0.08048664016431273,
  'rel_effect_size_ci_lower': -0.019515294044061937,
  'rel_effect_size_ci_upper': 0.1906880061278886,
  'statistic': 1.5647028839586707,
  'treatment': 0.5730905412240769},
 {'control': 5.241028175976273,
  'effect_size': 0.4890831037404775,
  'effect_size_ci_lower': -0.13261881482742033,
  'effect_size_ci_upper': 1.1107850223083753,
  'metric': 'revenue_per_user',
  'pvalue': np.float64(0.1230698855425058),
  'rel_effect_size': 0.09331815958981626,
  'rel_effect_size_ci_lower': -0.02373770894855798,
  'rel_effect_size_ci_upper': 0.22440926894909308,
  'statistic': 1.5423440700784083,
  'treatment': 5.73011127971675})

Source code in src/tea_tasting/experiment.py

@tea_tasting.utils._cache_method
def to_dicts(self) -> tuple[dict[str, object], ...]:
    """Convert the result to a sequence of dictionaries.

    Examples:
        ```pycon
        >>> import pprint
        >>> import tea_tasting as tt

        >>> experiment = tt.Experiment(
        ...     orders_per_user=tt.Mean("orders"),
        ...     revenue_per_user=tt.Mean("revenue"),
        ... )
        >>> data = tt.make_users_data(seed=42)
        >>> result = experiment.analyze(data)
        >>> pprint.pprint(result.to_dicts())
        ({'control': 0.5304003954522986,
          'effect_size': 0.04269014577177832,
          'effect_size_ci_lower': -0.010800201598205515,
          'effect_size_ci_upper': 0.09618049314176216,
          'metric': 'orders_per_user',
          'pvalue': np.float64(0.11773177998716214),
          'rel_effect_size': 0.08048664016431273,
          'rel_effect_size_ci_lower': -0.019515294044061937,
          'rel_effect_size_ci_upper': 0.1906880061278886,
          'statistic': 1.5647028839586707,
          'treatment': 0.5730905412240769},
         {'control': 5.241028175976273,
          'effect_size': 0.4890831037404775,
          'effect_size_ci_lower': -0.13261881482742033,
          'effect_size_ci_upper': 1.1107850223083753,
          'metric': 'revenue_per_user',
          'pvalue': np.float64(0.1230698855425058),
          'rel_effect_size': 0.09331815958981626,
          'rel_effect_size_ci_lower': -0.02373770894855798,
          'rel_effect_size_ci_upper': 0.22440926894909308,
          'statistic': 1.5423440700784083,
          'treatment': 5.73011127971675})

        ```
    """
    return tuple(
        {"metric": k} | (v if isinstance(v, dict) else v._asdict())
        for k, v in self.items()
    )  # type: ignore

to_html(keys=None, formatter=get_and_format_num, *, max_rows=None, indent=None)

Convert the object to HTML.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None
indent	str | None	Whitespace to insert for each indentation level. If None, do not indent.	None

Returns:

Type	Description
str	A table with results rendered as HTML.

Source code in src/tea_tasting/utils.py

def to_html(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
    indent: str | None = None,
) -> str:
    """Convert the object to HTML.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.
        indent: Whitespace to insert for each indentation level. If `None`,
            do not indent.

    Returns:
        A table with results rendered as HTML.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    table = ET.Element(
        "table",
        {"class": "dataframe", "style": "text-align: right;"},
    )
    thead = ET.SubElement(table, "thead")
    thead_tr = ET.SubElement(thead, "tr")
    for key in keys:
        th = ET.SubElement(thead_tr, "th")
        th.text = key
    tbody = ET.SubElement(table, "tbody")
    for pretty_dict in self.to_pretty_dicts(keys, formatter, max_rows=max_rows):
        tr = ET.SubElement(tbody, "tr")
        for key in keys:
            td = ET.SubElement(tr, "td")
            td.text = pretty_dict[key]
    if indent is not None:
        ET.indent(table, space=indent)
    return ET.tostring(table, encoding="unicode", method="html")

to_pandas()

Convert the object to a Pandas DataFrame.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_pandas(self) -> pd.DataFrame:
    """Convert the object to a Pandas DataFrame."""
    import pandas as pd
    return pd.DataFrame.from_records(self.to_dicts())

to_polars()

Convert the object to a Polars DataFrame.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_polars(self) -> pl.DataFrame:
    """Convert the object to a Polars DataFrame."""
    import polars as pl
    return pl.from_dicts(self.to_dicts())

to_pretty_dicts(keys=None, formatter=get_and_format_num, *, max_rows=None)

Convert the object to a list of dictionaries with formatted values.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None

Returns:

Type	Description
list[dict[str, str]]	List of dictionaries with formatted values.

Source code in src/tea_tasting/utils.py

def to_pretty_dicts(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
) -> list[dict[str, str]]:
    """Convert the object to a list of dictionaries with formatted values.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.

    Returns:
        List of dictionaries with formatted values.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    dicts = self.to_dicts()
    if max_rows <= 0 or len(dicts) <= max_rows:
        return [{key: formatter(data, key) for key in keys} for data in dicts]

    bottom = max_rows // 2
    top = max_rows - bottom
    return (
        [{key: formatter(data, key) for key in keys} for data in dicts[:top]] +
        [dict.fromkeys(keys, "…")] +
        [{key: formatter(data, key) for key in keys} for data in dicts[-bottom:]]
    )

to_string(keys=None, formatter=get_and_format_num, *, max_rows=None)

Convert the object to a string.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None

Returns:

Type	Description
str	A table with results rendered as string.

Source code in src/tea_tasting/utils.py

def to_string(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
) -> str:
    """Convert the object to a string.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.

    Returns:
        A table with results rendered as string.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    pretty_dicts = self.to_pretty_dicts(keys, formatter, max_rows=max_rows)
    widths = {key: len(key) for key in keys}
    for pretty_dict in pretty_dicts:
        for key in keys:
            widths[key] = max(widths[key], len(pretty_dict[key]))

    sep = " "
    rows = [sep.join(key.rjust(widths[key]) for key in keys)]
    rows.extend(
        sep.join(pretty_dict[key].rjust(widths[key]) for key in keys)
        for pretty_dict in pretty_dicts
    )
    return "\n".join(rows)

with_defaults(*, keys=None, max_rows=None)

Copies the object and sets the new default parameters.

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	New default keys for the methods to_pretty_dicts, to_string, and to_html.	None
max_rows	int | None	New default max_rows for the methods to_pretty_dicts, to_string, and to_html.	None

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default keys.

Source code in src/tea_tasting/utils.py

def with_defaults(
    self: DictsReprMixinT,
    *,
    keys: Sequence[str] | None = None,
    max_rows: int | None = None,
) -> DictsReprMixinT:
    """Copies the object and sets the new default parameters.

    Args:
        keys: New default `keys` for the methods `to_pretty_dicts`, `to_string`,
            and `to_html`.
        max_rows: New default `max_rows` for the methods `to_pretty_dicts`,
            `to_string`, and `to_html`.

    Returns:
        A copy of the object with the new default keys.
    """
    new_instance = self.__class__.__new__(self.__class__)
    new_instance.__dict__.update(self.__dict__)
    new_instance._cache = None
    if keys is not None:
        new_instance.default_keys = keys
    if max_rows is not None:
        new_instance.default_max_rows = max_rows
    return new_instance

with_keys(keys)

Copies the object and sets the new default keys.

Parameters:

Name	Type	Description	Default
keys	Sequence[str]	New default keys for the methods to_pretty_dicts, to_string, and to_html.	required

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default keys.

Source code in src/tea_tasting/utils.py

def with_keys(self: DictsReprMixinT, keys: Sequence[str]) -> DictsReprMixinT:
    """Copies the object and sets the new default `keys`.

    Args:
        keys: New default `keys` for the methods `to_pretty_dicts`, `to_string`,
            and `to_html`.

    Returns:
        A copy of the object with the new default `keys`.
    """
    return self.with_defaults(keys=keys)

with_max_rows(max_rows)

Copies the object and sets the new default max_rows.

Parameters:

Name	Type	Description	Default
max_rows	int	New default max_rows for the methods to_pretty_dicts, to_string, and to_html.	required

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default max_rows.

Source code in src/tea_tasting/utils.py

def with_max_rows(self: DictsReprMixinT, max_rows: int) -> DictsReprMixinT:
    """Copies the object and sets the new default `max_rows`.

    Args:
        max_rows: New default `max_rows` for the methods `to_pretty_dicts`,
            `to_string`, and `to_html`.

    Returns:
        A copy of the object with the new default `max_rows`.
    """
    return self.with_defaults(max_rows=max_rows)

ExperimentResults

Bases: DictsReprMixin, UserDict[tuple[object, object], ExperimentResult]

Experiment results for multiple pairs of variants.

to_arrow()

Convert the object to a PyArrow Table.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_arrow(self) -> pa.Table:
    """Convert the object to a PyArrow Table."""
    return pa.Table.from_pylist(self.to_dicts())

to_dicts()

Convert the results to a sequence of dictionaries.

Source code in src/tea_tasting/experiment.py

@tea_tasting.utils._cache_method
def to_dicts(self) -> tuple[dict[str, object], ...]:
    """Convert the results to a sequence of dictionaries."""
    return tuple(
        {"variants": str(variants)} | metric_result
        for variants, experiment_result in self.items()
        for metric_result in experiment_result.to_dicts()
    )

to_html(keys=None, formatter=get_and_format_num, *, max_rows=None, indent=None)

Convert the object to HTML.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None
indent	str | None	Whitespace to insert for each indentation level. If None, do not indent.	None

Returns:

Type	Description
str	A table with results rendered as HTML.

Source code in src/tea_tasting/utils.py

def to_html(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
    indent: str | None = None,
) -> str:
    """Convert the object to HTML.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.
        indent: Whitespace to insert for each indentation level. If `None`,
            do not indent.

    Returns:
        A table with results rendered as HTML.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    table = ET.Element(
        "table",
        {"class": "dataframe", "style": "text-align: right;"},
    )
    thead = ET.SubElement(table, "thead")
    thead_tr = ET.SubElement(thead, "tr")
    for key in keys:
        th = ET.SubElement(thead_tr, "th")
        th.text = key
    tbody = ET.SubElement(table, "tbody")
    for pretty_dict in self.to_pretty_dicts(keys, formatter, max_rows=max_rows):
        tr = ET.SubElement(tbody, "tr")
        for key in keys:
            td = ET.SubElement(tr, "td")
            td.text = pretty_dict[key]
    if indent is not None:
        ET.indent(table, space=indent)
    return ET.tostring(table, encoding="unicode", method="html")

to_pandas()

Convert the object to a Pandas DataFrame.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_pandas(self) -> pd.DataFrame:
    """Convert the object to a Pandas DataFrame."""
    import pandas as pd
    return pd.DataFrame.from_records(self.to_dicts())

to_polars()

Convert the object to a Polars DataFrame.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_polars(self) -> pl.DataFrame:
    """Convert the object to a Polars DataFrame."""
    import polars as pl
    return pl.from_dicts(self.to_dicts())

to_pretty_dicts(keys=None, formatter=get_and_format_num, *, max_rows=None)

Convert the object to a list of dictionaries with formatted values.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None

Returns:

Type	Description
list[dict[str, str]]	List of dictionaries with formatted values.

Source code in src/tea_tasting/utils.py

def to_pretty_dicts(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
) -> list[dict[str, str]]:
    """Convert the object to a list of dictionaries with formatted values.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.

    Returns:
        List of dictionaries with formatted values.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    dicts = self.to_dicts()
    if max_rows <= 0 or len(dicts) <= max_rows:
        return [{key: formatter(data, key) for key in keys} for data in dicts]

    bottom = max_rows // 2
    top = max_rows - bottom
    return (
        [{key: formatter(data, key) for key in keys} for data in dicts[:top]] +
        [dict.fromkeys(keys, "…")] +
        [{key: formatter(data, key) for key in keys} for data in dicts[-bottom:]]
    )

to_string(keys=None, formatter=get_and_format_num, *, max_rows=None)

Convert the object to a string.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None

Returns:

Type	Description
str	A table with results rendered as string.

Source code in src/tea_tasting/utils.py

def to_string(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
) -> str:
    """Convert the object to a string.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.

    Returns:
        A table with results rendered as string.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    pretty_dicts = self.to_pretty_dicts(keys, formatter, max_rows=max_rows)
    widths = {key: len(key) for key in keys}
    for pretty_dict in pretty_dicts:
        for key in keys:
            widths[key] = max(widths[key], len(pretty_dict[key]))

    sep = " "
    rows = [sep.join(key.rjust(widths[key]) for key in keys)]
    rows.extend(
        sep.join(pretty_dict[key].rjust(widths[key]) for key in keys)
        for pretty_dict in pretty_dicts
    )
    return "\n".join(rows)

with_defaults(*, keys=None, max_rows=None)

Copies the object and sets the new default parameters.

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	New default keys for the methods to_pretty_dicts, to_string, and to_html.	None
max_rows	int | None	New default max_rows for the methods to_pretty_dicts, to_string, and to_html.	None

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default keys.

Source code in src/tea_tasting/utils.py

def with_defaults(
    self: DictsReprMixinT,
    *,
    keys: Sequence[str] | None = None,
    max_rows: int | None = None,
) -> DictsReprMixinT:
    """Copies the object and sets the new default parameters.

    Args:
        keys: New default `keys` for the methods `to_pretty_dicts`, `to_string`,
            and `to_html`.
        max_rows: New default `max_rows` for the methods `to_pretty_dicts`,
            `to_string`, and `to_html`.

    Returns:
        A copy of the object with the new default keys.
    """
    new_instance = self.__class__.__new__(self.__class__)
    new_instance.__dict__.update(self.__dict__)
    new_instance._cache = None
    if keys is not None:
        new_instance.default_keys = keys
    if max_rows is not None:
        new_instance.default_max_rows = max_rows
    return new_instance

with_keys(keys)

Copies the object and sets the new default keys.

Parameters:

Name	Type	Description	Default
keys	Sequence[str]	New default keys for the methods to_pretty_dicts, to_string, and to_html.	required

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default keys.

Source code in src/tea_tasting/utils.py

def with_keys(self: DictsReprMixinT, keys: Sequence[str]) -> DictsReprMixinT:
    """Copies the object and sets the new default `keys`.

    Args:
        keys: New default `keys` for the methods `to_pretty_dicts`, `to_string`,
            and `to_html`.

    Returns:
        A copy of the object with the new default `keys`.
    """
    return self.with_defaults(keys=keys)

with_max_rows(max_rows)

Copies the object and sets the new default max_rows.

Parameters:

Name	Type	Description	Default
max_rows	int	New default max_rows for the methods to_pretty_dicts, to_string, and to_html.	required

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default max_rows.

Source code in src/tea_tasting/utils.py

def with_max_rows(self: DictsReprMixinT, max_rows: int) -> DictsReprMixinT:
    """Copies the object and sets the new default `max_rows`.

    Args:
        max_rows: New default `max_rows` for the methods `to_pretty_dicts`,
            `to_string`, and `to_html`.

    Returns:
        A copy of the object with the new default `max_rows`.
    """
    return self.with_defaults(max_rows=max_rows)

SimulationResults

Bases: DictsReprMixin, UserList[ExperimentResult]

Simulation results.

Simulations are not enumerated for better performance.

to_arrow()

Convert the object to a PyArrow Table.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_arrow(self) -> pa.Table:
    """Convert the object to a PyArrow Table."""
    return pa.Table.from_pylist(self.to_dicts())

to_dicts()

Convert the results to a sequence of dictionaries.

Source code in src/tea_tasting/experiment.py

@tea_tasting.utils._cache_method
def to_dicts(self) -> tuple[dict[str, object], ...]:
    """Convert the results to a sequence of dictionaries."""
    return tuple(itertools.chain.from_iterable(
        experiment_result.to_dicts()
        for experiment_result in self
    ))

to_html(keys=None, formatter=get_and_format_num, *, max_rows=None, indent=None)

Convert the object to HTML.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None
indent	str | None	Whitespace to insert for each indentation level. If None, do not indent.	None

Returns:

Type	Description
str	A table with results rendered as HTML.

Source code in src/tea_tasting/utils.py

def to_html(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
    indent: str | None = None,
) -> str:
    """Convert the object to HTML.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.
        indent: Whitespace to insert for each indentation level. If `None`,
            do not indent.

    Returns:
        A table with results rendered as HTML.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    table = ET.Element(
        "table",
        {"class": "dataframe", "style": "text-align: right;"},
    )
    thead = ET.SubElement(table, "thead")
    thead_tr = ET.SubElement(thead, "tr")
    for key in keys:
        th = ET.SubElement(thead_tr, "th")
        th.text = key
    tbody = ET.SubElement(table, "tbody")
    for pretty_dict in self.to_pretty_dicts(keys, formatter, max_rows=max_rows):
        tr = ET.SubElement(tbody, "tr")
        for key in keys:
            td = ET.SubElement(tr, "td")
            td.text = pretty_dict[key]
    if indent is not None:
        ET.indent(table, space=indent)
    return ET.tostring(table, encoding="unicode", method="html")

to_pandas()

Convert the object to a Pandas DataFrame.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_pandas(self) -> pd.DataFrame:
    """Convert the object to a Pandas DataFrame."""
    import pandas as pd
    return pd.DataFrame.from_records(self.to_dicts())

to_polars()

Convert the object to a Polars DataFrame.

Source code in src/tea_tasting/utils.py

@_cache_method
def to_polars(self) -> pl.DataFrame:
    """Convert the object to a Polars DataFrame."""
    import polars as pl
    return pl.from_dicts(self.to_dicts())

to_pretty_dicts(keys=None, formatter=get_and_format_num, *, max_rows=None)

Convert the object to a list of dictionaries with formatted values.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None

Returns:

Type	Description
list[dict[str, str]]	List of dictionaries with formatted values.

Source code in src/tea_tasting/utils.py

def to_pretty_dicts(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
) -> list[dict[str, str]]:
    """Convert the object to a list of dictionaries with formatted values.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.

    Returns:
        List of dictionaries with formatted values.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    dicts = self.to_dicts()
    if max_rows <= 0 or len(dicts) <= max_rows:
        return [{key: formatter(data, key) for key in keys} for data in dicts]

    bottom = max_rows // 2
    top = max_rows - bottom
    return (
        [{key: formatter(data, key) for key in keys} for data in dicts[:top]] +
        [dict.fromkeys(keys, "…")] +
        [{key: formatter(data, key) for key in keys} for data in dicts[-bottom:]]
    )

to_string(keys=None, formatter=get_and_format_num, *, max_rows=None)

Convert the object to a string.

Default formatting rules:

• If a name starts with "rel_" or equals to "power" consider it a percentage value. Round percentage values to 2 significant digits, multiply by 100 and add "%".
• Round other values to 3 significant values.
• If value is less than 0.001 or is greater than or equal to 10_000_000, format it in exponential presentation.
• If a name ends with "_ci", consider it a confidence interval. Look up for attributes "{name}_lower" and "{name}_upper", and format the interval as "[{lower_bound}, {upper_bound}]".

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	Keys to convert. If a key is not defined in the dictionary it's assumed to be None.	None
formatter	Callable[[dict[str, object], str], str]	Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value.	get_and_format_num
max_rows	int | None	Maximum number of rows to convert. If None, the default value will be used. If 0 or less, all rows will be converted.	None

Returns:

Type	Description
str	A table with results rendered as string.

Source code in src/tea_tasting/utils.py

def to_string(
    self,
    keys: Sequence[str] | None = None,
    formatter: Callable[[dict[str, object], str], str] = get_and_format_num,
    *,
    max_rows: int | None = None,
) -> str:
    """Convert the object to a string.

    Default formatting rules:

    - If a name starts with `"rel_"` or equals to `"power"` consider it
        a percentage value. Round percentage values to 2 significant digits,
        multiply by `100` and add `"%"`.
    - Round other values to 3 significant values.
    - If value is less than `0.001` or is greater than or equal to `10_000_000`,
        format it in exponential presentation.
    - If a name ends with `"_ci"`, consider it a confidence interval.
        Look up for attributes `"{name}_lower"` and `"{name}_upper"`,
        and format the interval as `"[{lower_bound}, {upper_bound}]"`.

    Args:
        keys: Keys to convert. If a key is not defined in the dictionary
            it's assumed to be `None`.
        formatter: Custom formatter function. It should accept a dictionary
            of metric result attributes and an attribute name, and return
            a formatted attribute value.
        max_rows: Maximum number of rows to convert.
            If `None`, the default value will be used.
            If `0` or less, all rows will be converted.

    Returns:
        A table with results rendered as string.
    """
    if keys is None:
        keys = self.default_keys
    if max_rows is None:
        max_rows = self.default_max_rows

    pretty_dicts = self.to_pretty_dicts(keys, formatter, max_rows=max_rows)
    widths = {key: len(key) for key in keys}
    for pretty_dict in pretty_dicts:
        for key in keys:
            widths[key] = max(widths[key], len(pretty_dict[key]))

    sep = " "
    rows = [sep.join(key.rjust(widths[key]) for key in keys)]
    rows.extend(
        sep.join(pretty_dict[key].rjust(widths[key]) for key in keys)
        for pretty_dict in pretty_dicts
    )
    return "\n".join(rows)

with_defaults(*, keys=None, max_rows=None)

Copies the object and sets the new default parameters.

Parameters:

Name	Type	Description	Default
keys	Sequence[str] | None	New default keys for the methods to_pretty_dicts, to_string, and to_html.	None
max_rows	int | None	New default max_rows for the methods to_pretty_dicts, to_string, and to_html.	None

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default keys.

Source code in src/tea_tasting/utils.py

def with_defaults(
    self: DictsReprMixinT,
    *,
    keys: Sequence[str] | None = None,
    max_rows: int | None = None,
) -> DictsReprMixinT:
    """Copies the object and sets the new default parameters.

    Args:
        keys: New default `keys` for the methods `to_pretty_dicts`, `to_string`,
            and `to_html`.
        max_rows: New default `max_rows` for the methods `to_pretty_dicts`,
            `to_string`, and `to_html`.

    Returns:
        A copy of the object with the new default keys.
    """
    new_instance = self.__class__.__new__(self.__class__)
    new_instance.__dict__.update(self.__dict__)
    new_instance._cache = None
    if keys is not None:
        new_instance.default_keys = keys
    if max_rows is not None:
        new_instance.default_max_rows = max_rows
    return new_instance

with_keys(keys)

Copies the object and sets the new default keys.

Parameters:

Name	Type	Description	Default
keys	Sequence[str]	New default keys for the methods to_pretty_dicts, to_string, and to_html.	required

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default keys.

Source code in src/tea_tasting/utils.py

def with_keys(self: DictsReprMixinT, keys: Sequence[str]) -> DictsReprMixinT:
    """Copies the object and sets the new default `keys`.

    Args:
        keys: New default `keys` for the methods `to_pretty_dicts`, `to_string`,
            and `to_html`.

    Returns:
        A copy of the object with the new default `keys`.
    """
    return self.with_defaults(keys=keys)

with_max_rows(max_rows)

Copies the object and sets the new default max_rows.

Parameters:

Name	Type	Description	Default
max_rows	int	New default max_rows for the methods to_pretty_dicts, to_string, and to_html.	required

Returns:

Type	Description
DictsReprMixinT	A copy of the object with the new default max_rows.

Source code in src/tea_tasting/utils.py

def with_max_rows(self: DictsReprMixinT, max_rows: int) -> DictsReprMixinT:
    """Copies the object and sets the new default `max_rows`.

    Args:
        max_rows: New default `max_rows` for the methods `to_pretty_dicts`,
            `to_string`, and `to_html`.

    Returns:
        A copy of the object with the new default `max_rows`.
    """
    return self.with_defaults(max_rows=max_rows)