tea_tasting.metrics.mean

Metrics for the analysis of means.

Mean(value, covariate=None, *, alternative=None, confidence_level=None, equal_var=None, use_t=None, alpha=None, ratio=None, power=None, effect_size=None, rel_effect_size=None, n_obs=None)

Bases: RatioOfMeans

Metric for the analysis of means.

Parameters:

Name	Type	Description	Default
value	str	Metric value column name.	required
covariate	str | None	Metric covariate column name.	None
alternative	Literal['two-sided', 'greater', 'less'] | None	Alternative hypothesis: "two-sided": the means are unequal, "greater": the mean in the treatment variant is greater than the mean in the control variant, "less": the mean in the treatment variant is less than the mean in the control variant.	None
confidence_level	float | None	Confidence level for the confidence interval.	None
equal_var	bool | None	Defines whether equal variance is assumed. If True, pooled variance is used for the calculation of the standard error of the difference between two means.	None
use_t	bool | None	Defines whether to use the Student's t-distribution (True) or the Normal distribution (False).	None
alpha	float | None	Significance level. Only for the analysis of power.	None
ratio	float | int | None	Ratio of the number of observations in the treatment relative to the control. Only for the analysis of power.	None
power	float | None	Statistical power. Only for the analysis of power.	None
effect_size	float | int | Sequence[float | int] | None	Absolute effect size. Difference between the two means. Only for the analysis of power.	None
rel_effect_size	float | Sequence[float] | None	Relative effect size. Difference between the two means, divided by the control mean. Only for the analysis of power.	None
n_obs	int | Sequence[int] | None	Number of observations in the control and in the treatment together. Only for the analysis of power.	None

Parameter defaults

Defaults for parameters alpha, alternative, confidence_level, equal_var, n_obs, power, ratio, and use_t can be changed using the config_context and set_context functions. See the Global configuration reference for details.

References

• Deng, A., Knoblich, U., & Lu, J. (2018). Applying the Delta Method in Metric Analytics: A Practical Guide with Novel Ideas.
• Deng, A., Xu, Y., Kohavi, R., & Walker, T. (2013). Improving the Sensitivity of Online Controlled Experiments by Utilizing Pre-Experiment Data.

Examples:

>>> import tea_tasting as tt

>>> experiment = tt.Experiment(
...     orders_per_user=tt.Mean("orders"),
...     revenue_per_user=tt.Mean("revenue"),
... )
>>> data = tt.make_users_data(seed=42)
>>> result = experiment.analyze(data)
>>> print(result)
          metric control treatment rel_effect_size rel_effect_size_ci pvalue
 orders_per_user   0.530     0.573            8.0%       [-2.0%, 19%]  0.118
revenue_per_user    5.24      5.73            9.3%       [-2.4%, 22%]  0.123

With CUPED:

>>> experiment = tt.Experiment(
...     orders_per_user=tt.Mean("orders", "orders_covariate"),
...     revenue_per_user=tt.Mean("revenue", "revenue_covariate"),
... )
>>> data = tt.make_users_data(seed=42, covariates=True)
>>> result = experiment.analyze(data)
>>> print(result)
          metric control treatment rel_effect_size rel_effect_size_ci  pvalue
 orders_per_user   0.523     0.581             11%        [2.9%, 20%] 0.00733
revenue_per_user    5.12      5.85             14%        [3.8%, 26%] 0.00674

Power analysis:

>>> data = tt.make_users_data(
...     seed=42,
...     sessions_uplift=0,
...     orders_uplift=0,
...     revenue_uplift=0,
...     covariates=True,
... )
>>> orders_per_user = tt.Mean(
...     "orders",
...     "orders_covariate",
...     n_obs=(10_000, 20_000),
... )
>>> # Solve for effect size.
>>> print(orders_per_user.solve_power(data))
power effect_size rel_effect_size n_obs
  80%      0.0374            7.2% 10000
  80%      0.0264            5.1% 20000

>>> orders_per_user = tt.Mean(
...     "orders",
...     "orders_covariate",
...     rel_effect_size=0.05,
... )
>>> # Solve for the total number of observations.
>>> print(orders_per_user.solve_power(data, "n_obs"))
power effect_size rel_effect_size n_obs
  80%      0.0260            5.0% 20733

>>> orders_per_user = tt.Mean(
...     "orders",
...     "orders_covariate",
...     rel_effect_size=0.1,
... )
>>> # Solve for power. Infer number of observations from the sample.
>>> print(orders_per_user.solve_power(data, "power"))
power effect_size rel_effect_size n_obs
  69%      0.0519             10%  4000

Source code in src/tea_tasting/metrics/mean.py

def __init__(  # noqa: PLR0913
    self,
    value: str,
    covariate: str | None = None,
    *,
    alternative: Literal["two-sided", "greater", "less"] | None = None,
    confidence_level: float | None = None,
    equal_var: bool | None = None,
    use_t: bool | None = None,
    alpha: float | None = None,
    ratio: float | int | None = None,
    power: float | None = None,
    effect_size: float | int | Sequence[float | int] | None = None,
    rel_effect_size: float | Sequence[float] | None = None,
    n_obs: int | Sequence[int] | None = None,
) -> None:
    """Metric for the analysis of means.

    Args:
        value: Metric value column name.
        covariate: Metric covariate column name.
        alternative: Alternative hypothesis:

            - `"two-sided"`: the means are unequal,
            - `"greater"`: the mean in the treatment variant is greater than the mean
                in the control variant,
            - `"less"`: the mean in the treatment variant is less than the mean
                in the control variant.

        confidence_level: Confidence level for the confidence interval.
        equal_var: Defines whether equal variance is assumed. If `True`,
            pooled variance is used for the calculation of the standard error
            of the difference between two means.
        use_t: Defines whether to use the Student's t-distribution (`True`) or
            the Normal distribution (`False`).
        alpha: Significance level. Only for the analysis of power.
        ratio: Ratio of the number of observations in the treatment
            relative to the control. Only for the analysis of power.
        power: Statistical power. Only for the analysis of power.
        effect_size: Absolute effect size. Difference between the two means.
            Only for the analysis of power.
        rel_effect_size: Relative effect size. Difference between the two means,
            divided by the control mean. Only for the analysis of power.
        n_obs: Number of observations in the control and in the treatment together.
            Only for the analysis of power.

    Parameter defaults:
        Defaults for parameters `alpha`, `alternative`, `confidence_level`,
        `equal_var`, `n_obs`, `power`, `ratio`, and `use_t` can be changed
        using the `config_context` and `set_context` functions.
        See the [Global configuration](https://tea-tasting.e10v.me/api/config/)
        reference for details.

    References:
        - [Deng, A., Knoblich, U., & Lu, J. (2018). Applying the Delta Method in Metric Analytics: A Practical Guide with Novel Ideas](https://alexdeng.github.io/public/files/kdd2018-dm.pdf).
        - [Deng, A., Xu, Y., Kohavi, R., & Walker, T. (2013). Improving the Sensitivity of Online Controlled Experiments by Utilizing Pre-Experiment Data](https://exp-platform.com/Documents/2013-02-CUPED-ImprovingSensitivityOfControlledExperiments.pdf).

    Examples:
        ```pycon
        >>> import tea_tasting as tt

        >>> experiment = tt.Experiment(
        ...     orders_per_user=tt.Mean("orders"),
        ...     revenue_per_user=tt.Mean("revenue"),
        ... )
        >>> data = tt.make_users_data(seed=42)
        >>> result = experiment.analyze(data)
        >>> print(result)
                  metric control treatment rel_effect_size rel_effect_size_ci pvalue
         orders_per_user   0.530     0.573            8.0%       [-2.0%, 19%]  0.118
        revenue_per_user    5.24      5.73            9.3%       [-2.4%, 22%]  0.123

        ```

        With CUPED:

        ```pycon
        >>> experiment = tt.Experiment(
        ...     orders_per_user=tt.Mean("orders", "orders_covariate"),
        ...     revenue_per_user=tt.Mean("revenue", "revenue_covariate"),
        ... )
        >>> data = tt.make_users_data(seed=42, covariates=True)
        >>> result = experiment.analyze(data)
        >>> print(result)
                  metric control treatment rel_effect_size rel_effect_size_ci  pvalue
         orders_per_user   0.523     0.581             11%        [2.9%, 20%] 0.00733
        revenue_per_user    5.12      5.85             14%        [3.8%, 26%] 0.00674

        ```

        Power analysis:

        ```pycon
        >>> data = tt.make_users_data(
        ...     seed=42,
        ...     sessions_uplift=0,
        ...     orders_uplift=0,
        ...     revenue_uplift=0,
        ...     covariates=True,
        ... )
        >>> orders_per_user = tt.Mean(
        ...     "orders",
        ...     "orders_covariate",
        ...     n_obs=(10_000, 20_000),
        ... )
        >>> # Solve for effect size.
        >>> print(orders_per_user.solve_power(data))
        power effect_size rel_effect_size n_obs
          80%      0.0374            7.2% 10000
          80%      0.0264            5.1% 20000

        >>> orders_per_user = tt.Mean(
        ...     "orders",
        ...     "orders_covariate",
        ...     rel_effect_size=0.05,
        ... )
        >>> # Solve for the total number of observations.
        >>> print(orders_per_user.solve_power(data, "n_obs"))
        power effect_size rel_effect_size n_obs
          80%      0.0260            5.0% 20733

        >>> orders_per_user = tt.Mean(
        ...     "orders",
        ...     "orders_covariate",
        ...     rel_effect_size=0.1,
        ... )
        >>> # Solve for power. Infer number of observations from the sample.
        >>> print(orders_per_user.solve_power(data, "power"))
        power effect_size rel_effect_size n_obs
          69%      0.0519             10%  4000

        ```
    """  # noqa: E501
    super().__init__(
        numer=value,
        denom=None,
        numer_covariate=covariate,
        denom_covariate=None,
        alternative=alternative,
        confidence_level=confidence_level,
        equal_var=equal_var,
        use_t=use_t,
        alpha=alpha,
        ratio=ratio,
        power=power,
        effect_size=effect_size,
        rel_effect_size=rel_effect_size,
        n_obs=n_obs,
    )
    self.value = value
    self.covariate = covariate

aggr_cols: AggrCols property

Columns to be aggregated for a metric analysis.

analyze(data, control, treatment, variant=None)

Analyze a metric in an experiment.

Parameters:

Name	Type	Description	Default
data	IntoFrame | Table | dict[Any, Aggregates]	Experimental data.	required
control	Any	Control variant.	required
treatment	Any	Treatment variant.	required
variant	str | None	Variant column name.	None

Returns:

Type	Description
R	Analysis result.

Source code in src/tea_tasting/metrics/base.py

def analyze(
    self,
    data: narwhals.typing.IntoFrame | ibis.expr.types.Table | dict[
        Any, tea_tasting.aggr.Aggregates],
    control: Any,
    treatment: Any,
    variant: str | None = None,
) -> R:
    """Analyze a metric in an experiment.

    Args:
        data: Experimental data.
        control: Control variant.
        treatment: Treatment variant.
        variant: Variant column name.

    Returns:
        Analysis result.
    """
    aggr = aggregate_by_variants(
        data,
        aggr_cols=self.aggr_cols,
        variant=variant,
    )
    return self.analyze_aggregates(
        control=aggr[control],
        treatment=aggr[treatment],
    )

analyze_aggregates(control, treatment)

Analyze a metric in an experiment using aggregated statistics.

Parameters:

Name	Type	Description	Default
control	Aggregates	Control data.	required
treatment	Aggregates	Treatment data.	required

Returns:

Type	Description
MeanResult	Analysis result.

Source code in src/tea_tasting/metrics/mean.py

def analyze_aggregates(
    self,
    control: tea_tasting.aggr.Aggregates,
    treatment: tea_tasting.aggr.Aggregates,
) -> MeanResult:
    """Analyze a metric in an experiment using aggregated statistics.

    Args:
        control: Control data.
        treatment: Treatment data.

    Returns:
        Analysis result.
    """
    control = control.with_zero_div()
    treatment = treatment.with_zero_div()
    total = control + treatment
    covariate_coef = self._covariate_coef(total)
    covariate_mean = total.mean(self.numer_covariate) / total.mean(
        self.denom_covariate)
    return self._analyze_stats(
        contr_mean=self._metric_mean(control, covariate_coef, covariate_mean),
        contr_var=self._metric_var(control, covariate_coef),
        contr_count=control.count(),
        treat_mean=self._metric_mean(treatment, covariate_coef, covariate_mean),
        treat_var=self._metric_var(treatment, covariate_coef),
        treat_count=treatment.count(),
    )

solve_power(data, parameter='rel_effect_size')

Solve for a parameter of the power of a test.

Parameters:

Name	Type	Description	Default
data	IntoFrame | Table | Aggregates	Sample data.	required
parameter	Literal['power', 'effect_size', 'rel_effect_size', 'n_obs']	Parameter name.	'rel_effect_size'

Returns:

Type	Description
S	Power analysis result.

Source code in src/tea_tasting/metrics/base.py

def solve_power(
    self,
    data: (
        narwhals.typing.IntoFrame |
        ibis.expr.types.Table |
        tea_tasting.aggr.Aggregates
    ),
    parameter: Literal[
        "power", "effect_size", "rel_effect_size", "n_obs"] = "rel_effect_size",
) -> S:
    """Solve for a parameter of the power of a test.

    Args:
        data: Sample data.
        parameter: Parameter name.

    Returns:
        Power analysis result.
    """
    if not isinstance(data, tea_tasting.aggr.Aggregates):
        data = tea_tasting.aggr.read_aggregates(
            data=data,
            group_col=None,
            **self.aggr_cols._asdict(),
        )
    return self.solve_power_from_aggregates(data=data, parameter=parameter)

solve_power_from_aggregates(data, parameter='rel_effect_size')

Solve for a parameter of the power of a test.

Parameters:

Name	Type	Description	Default
data	Aggregates	Sample data.	required
parameter	Literal['power', 'effect_size', 'rel_effect_size', 'n_obs']	Parameter name.	'rel_effect_size'

Returns:

Type	Description
MeanPowerResults	Power analysis result.

Source code in src/tea_tasting/metrics/mean.py

def solve_power_from_aggregates(
    self,
    data: tea_tasting.aggr.Aggregates,
    parameter: Literal[
        "power", "effect_size", "rel_effect_size", "n_obs"] = "rel_effect_size",
) -> MeanPowerResults:
    """Solve for a parameter of the power of a test.

    Args:
        data: Sample data.
        parameter: Parameter name.

    Returns:
        Power analysis result.
    """
    tea_tasting.utils.check_scalar(
        parameter,
        "parameter",
        in_={"power", "effect_size", "rel_effect_size", "n_obs"},
    )

    data = data.with_zero_div()
    covariate_coef = self._covariate_coef(data)
    covariate_mean = data.mean(self.numer_covariate) / data.mean(
        self.denom_covariate)
    metric_mean = self._metric_mean(data, covariate_coef, covariate_mean)

    power, effect_size, rel_effect_size, n_obs = self._validate_power_parameters(
        metric_mean=metric_mean,
        sample_count=data.count(),
        parameter=parameter,
    )

    result = MeanPowerResults()
    for effect_size_i, rel_effect_size_i in zip(
        effect_size,
        rel_effect_size,
        strict=True,
    ):
        for n_obs_i in n_obs:
            parameter_value = self._solve_power_from_stats(
                sample_var=self._metric_var(data, covariate_coef),
                sample_count=n_obs_i,
                effect_size=effect_size_i,
                power=power,
            )
            result.append(MeanPowerResult(
                power=parameter_value if parameter == "power" else power,  # type: ignore
                effect_size=(
                    parameter_value
                    if parameter in {"effect_size", "rel_effect_size"}
                    else effect_size_i
                ),  # type: ignore
                rel_effect_size=(
                    parameter_value / metric_mean
                    if parameter in {"effect_size", "rel_effect_size"}
                    else rel_effect_size_i
                ),  # type: ignore
                n_obs=(
                    math.ceil(parameter_value)
                    if parameter == "n_obs"
                    else n_obs_i
                ),  # type: ignore
            ))

    return result

MeanPowerResult

Bases: NamedTuple

Power analysis results.

Attributes:

Name	Type	Description
power	float	Statistical power.
effect_size	float	Absolute effect size. Difference between the two means.
rel_effect_size	float	Relative effect size. Difference between the two means, divided by the control mean.
n_obs	float	Number of observations in the control and in the treatment together.

MeanResult

Bases: NamedTuple

Result of the analysis of means.

Attributes:

Name	Type	Description
control	float	Control mean.
treatment	float	Treatment mean.
effect_size	float	Absolute effect size. Difference between the two means.
effect_size_ci_lower	float	Lower bound of the absolute effect size confidence interval.
effect_size_ci_upper	float	Upper bound of the absolute effect size confidence interval.
rel_effect_size	float	Relative effect size. Difference between the two means, divided by the control mean.
rel_effect_size_ci_lower	float	Lower bound of the relative effect size confidence interval.
rel_effect_size_ci_upper	float	Upper bound of the relative effect size confidence interval.
pvalue	float	P-value
statistic	float	Statistic (standardized effect size).

RatioOfMeans(numer, denom=None, numer_covariate=None, denom_covariate=None, *, alternative=None, confidence_level=None, equal_var=None, use_t=None, alpha=None, ratio=None, power=None, effect_size=None, rel_effect_size=None, n_obs=None)

Bases: MetricBaseAggregated[MeanResult], PowerBaseAggregated[MeanPowerResults]

Metric for the analysis of ratios of means.

Parameters:

Name	Type	Description	Default
numer	str	Numerator column name.	required
denom	str | None	Denominator column name.	None
numer_covariate	str | None	Covariate numerator column name.	None
denom_covariate	str | None	Covariate denominator column name.	None
alternative	Literal['two-sided', 'greater', 'less'] | None	Alternative hypothesis: "two-sided": the means are unequal, "greater": the mean in the treatment variant is greater than the mean in the control variant, "less": the mean in the treatment variant is less than the mean in the control variant.	None
confidence_level	float | None	Confidence level for the confidence interval.	None
equal_var	bool | None	Defines whether equal variance is assumed. If True, pooled variance is used for the calculation of the standard error of the difference between two means.	None
use_t	bool | None	Defines whether to use the Student's t-distribution (True) or the Normal distribution (False).	None
alpha	float | None	Significance level. Only for the analysis of power.	None
ratio	float | int | None	Ratio of the number of observations in the treatment relative to the control. Only for the analysis of power.	None
power	float | None	Statistical power. Only for the analysis of power.	None
effect_size	float | int | Sequence[float | int] | None	Absolute effect size. Difference between the two means. Only for the analysis of power.	None
rel_effect_size	float | Sequence[float] | None	Relative effect size. Difference between the two means, divided by the control mean. Only for the analysis of power.	None
n_obs	int | Sequence[int] | None	Number of observations in the control and in the treatment together. Only for the analysis of power.	None

Parameter defaults

Defaults for parameters alpha, alternative, confidence_level, equal_var, n_obs, power, ratio, and use_t can be changed using the config_context and set_context functions. See the Global configuration reference for details.

References

• Deng, A., Knoblich, U., & Lu, J. (2018). Applying the Delta Method in Metric Analytics: A Practical Guide with Novel Ideas.
• Deng, A., Xu, Y., Kohavi, R., & Walker, T. (2013). Improving the Sensitivity of Online Controlled Experiments by Utilizing Pre-Experiment Data.

Examples:

>>> import tea_tasting as tt

>>> experiment = tt.Experiment(
...     orders_per_session=tt.RatioOfMeans("orders", "sessions"),
... )
>>> data = tt.make_users_data(seed=42)
>>> result = experiment.analyze(data)
>>> print(result)
            metric control treatment rel_effect_size rel_effect_size_ci pvalue
orders_per_session   0.266     0.289            8.8%      [-0.89%, 19%] 0.0762

With CUPED:

>>> experiment = tt.Experiment(
...     orders_per_session=tt.RatioOfMeans(
...         "orders",
...         "sessions",
...         "orders_covariate",
...         "sessions_covariate",
...     ),
... )
>>> data = tt.make_users_data(seed=42, covariates=True)
>>> result = experiment.analyze(data)
>>> print(result)
            metric control treatment rel_effect_size rel_effect_size_ci  pvalue
orders_per_session   0.262     0.293             12%        [4.2%, 21%] 0.00229

Power analysis:

>>> data = tt.make_users_data(
...     seed=42,
...     sessions_uplift=0,
...     orders_uplift=0,
...     revenue_uplift=0,
...     covariates=True,
... )
>>> orders_per_session = tt.RatioOfMeans(
...     "orders",
...     "sessions",
...     "orders_covariate",
...     "sessions_covariate",
...     n_obs=(10_000, 20_000),
... )
>>> # Solve for effect size.
>>> print(orders_per_session.solve_power(data))
power effect_size rel_effect_size n_obs
  80%      0.0177            6.8% 10000
  80%      0.0125            4.8% 20000

>>> orders_per_session = tt.RatioOfMeans(
...     "orders",
...     "sessions",
...     "orders_covariate",
...     "sessions_covariate",
...     rel_effect_size=0.05,
... )
>>> # Solve for the total number of observations.
>>> print(orders_per_session.solve_power(data, "n_obs"))
power effect_size rel_effect_size n_obs
  80%      0.0130            5.0% 18515

>>> orders_per_session = tt.RatioOfMeans(
...     "orders",
...     "sessions",
...     "orders_covariate",
...     "sessions_covariate",
...     rel_effect_size=0.1,
... )
>>> # Solve for power. Infer number of observations from the sample.
>>> print(orders_per_session.solve_power(data, "power"))
power effect_size rel_effect_size n_obs
  74%      0.0261             10%  4000

Source code in src/tea_tasting/metrics/mean.py

def __init__(  # noqa: PLR0913
    self,
    numer: str,
    denom: str | None = None,
    numer_covariate: str | None = None,
    denom_covariate: str | None = None,
    *,
    alternative: Literal["two-sided", "greater", "less"] | None = None,
    confidence_level: float | None = None,
    equal_var: bool | None = None,
    use_t: bool | None = None,
    alpha: float | None = None,
    ratio: float | int | None = None,
    power: float | None = None,
    effect_size: float | int | Sequence[float | int] | None = None,
    rel_effect_size: float | Sequence[float] | None = None,
    n_obs: int | Sequence[int] | None = None,
) -> None:
    """Metric for the analysis of ratios of means.

    Args:
        numer: Numerator column name.
        denom: Denominator column name.
        numer_covariate: Covariate numerator column name.
        denom_covariate: Covariate denominator column name.
        alternative: Alternative hypothesis:

            - `"two-sided"`: the means are unequal,
            - `"greater"`: the mean in the treatment variant is greater than the mean
                in the control variant,
            - `"less"`: the mean in the treatment variant is less than the mean
                in the control variant.

        confidence_level: Confidence level for the confidence interval.
        equal_var: Defines whether equal variance is assumed. If `True`,
            pooled variance is used for the calculation of the standard error
            of the difference between two means.
        use_t: Defines whether to use the Student's t-distribution (`True`) or
            the Normal distribution (`False`).
        alpha: Significance level. Only for the analysis of power.
        ratio: Ratio of the number of observations in the treatment
            relative to the control. Only for the analysis of power.
        power: Statistical power. Only for the analysis of power.
        effect_size: Absolute effect size. Difference between the two means.
            Only for the analysis of power.
        rel_effect_size: Relative effect size. Difference between the two means,
            divided by the control mean. Only for the analysis of power.
        n_obs: Number of observations in the control and in the treatment together.
            Only for the analysis of power.

    Parameter defaults:
        Defaults for parameters `alpha`, `alternative`, `confidence_level`,
        `equal_var`, `n_obs`, `power`, `ratio`, and `use_t` can be changed
        using the `config_context` and `set_context` functions.
        See the [Global configuration](https://tea-tasting.e10v.me/api/config/)
        reference for details.

    References:
        - [Deng, A., Knoblich, U., & Lu, J. (2018). Applying the Delta Method in Metric Analytics: A Practical Guide with Novel Ideas](https://alexdeng.github.io/public/files/kdd2018-dm.pdf).
        - [Deng, A., Xu, Y., Kohavi, R., & Walker, T. (2013). Improving the Sensitivity of Online Controlled Experiments by Utilizing Pre-Experiment Data](https://exp-platform.com/Documents/2013-02-CUPED-ImprovingSensitivityOfControlledExperiments.pdf).

    Examples:
        ```pycon
        >>> import tea_tasting as tt

        >>> experiment = tt.Experiment(
        ...     orders_per_session=tt.RatioOfMeans("orders", "sessions"),
        ... )
        >>> data = tt.make_users_data(seed=42)
        >>> result = experiment.analyze(data)
        >>> print(result)
                    metric control treatment rel_effect_size rel_effect_size_ci pvalue
        orders_per_session   0.266     0.289            8.8%      [-0.89%, 19%] 0.0762

        ```

        With CUPED:

        ```pycon
        >>> experiment = tt.Experiment(
        ...     orders_per_session=tt.RatioOfMeans(
        ...         "orders",
        ...         "sessions",
        ...         "orders_covariate",
        ...         "sessions_covariate",
        ...     ),
        ... )
        >>> data = tt.make_users_data(seed=42, covariates=True)
        >>> result = experiment.analyze(data)
        >>> print(result)
                    metric control treatment rel_effect_size rel_effect_size_ci  pvalue
        orders_per_session   0.262     0.293             12%        [4.2%, 21%] 0.00229

        ```

        Power analysis:

        ```pycon
        >>> data = tt.make_users_data(
        ...     seed=42,
        ...     sessions_uplift=0,
        ...     orders_uplift=0,
        ...     revenue_uplift=0,
        ...     covariates=True,
        ... )
        >>> orders_per_session = tt.RatioOfMeans(
        ...     "orders",
        ...     "sessions",
        ...     "orders_covariate",
        ...     "sessions_covariate",
        ...     n_obs=(10_000, 20_000),
        ... )
        >>> # Solve for effect size.
        >>> print(orders_per_session.solve_power(data))
        power effect_size rel_effect_size n_obs
          80%      0.0177            6.8% 10000
          80%      0.0125            4.8% 20000

        >>> orders_per_session = tt.RatioOfMeans(
        ...     "orders",
        ...     "sessions",
        ...     "orders_covariate",
        ...     "sessions_covariate",
        ...     rel_effect_size=0.05,
        ... )
        >>> # Solve for the total number of observations.
        >>> print(orders_per_session.solve_power(data, "n_obs"))
        power effect_size rel_effect_size n_obs
          80%      0.0130            5.0% 18515

        >>> orders_per_session = tt.RatioOfMeans(
        ...     "orders",
        ...     "sessions",
        ...     "orders_covariate",
        ...     "sessions_covariate",
        ...     rel_effect_size=0.1,
        ... )
        >>> # Solve for power. Infer number of observations from the sample.
        >>> print(orders_per_session.solve_power(data, "power"))
        power effect_size rel_effect_size n_obs
          74%      0.0261             10%  4000

        ```
    """  # noqa: E501
    self.numer = tea_tasting.utils.check_scalar(numer, "numer", typ=str)
    self.denom = tea_tasting.utils.check_scalar(denom, "denom", typ=str | None)
    self.numer_covariate = tea_tasting.utils.check_scalar(
        numer_covariate, "numer_covariate", typ=str | None)
    self.denom_covariate = tea_tasting.utils.check_scalar(
        denom_covariate, "denom_covariate", typ=str | None)
    self.alternative = (
        tea_tasting.utils.auto_check(alternative, "alternative")
        if alternative is not None
        else tea_tasting.config.get_config("alternative")
    )
    self.confidence_level = (
        tea_tasting.utils.auto_check(confidence_level, "confidence_level")
        if confidence_level is not None
        else tea_tasting.config.get_config("confidence_level")
    )
    self.equal_var = (
        tea_tasting.utils.auto_check(equal_var, "equal_var")
        if equal_var is not None
        else tea_tasting.config.get_config("equal_var")
    )
    self.use_t = (
        tea_tasting.utils.auto_check(use_t, "use_t")
        if use_t is not None
        else tea_tasting.config.get_config("use_t")
    )
    self.alpha = (
        tea_tasting.utils.auto_check(alpha, "alpha")
        if alpha is not None
        else tea_tasting.config.get_config("alpha")
    )
    self.ratio = (
        tea_tasting.utils.auto_check(ratio, "ratio")
        if ratio is not None
        else tea_tasting.config.get_config("ratio")
    )
    self.power = (
        tea_tasting.utils.auto_check(power, "power")
        if power is not None
        else tea_tasting.config.get_config("power")
    )
    if effect_size is not None and rel_effect_size is not None:
        raise ValueError(
            "Both `effect_size` and `rel_effect_size` are not `None`. "
            "Only one of them should be defined.",
        )
    if isinstance(effect_size, Sequence):
        for x in effect_size:
            tea_tasting.utils.check_scalar(
                x, "effect_size", typ=float | int,
                gt=float("-inf"), lt=float("inf"), ne=0,
            )
    elif effect_size is not None:
        tea_tasting.utils.check_scalar(
            effect_size, "effect_size", typ=float | int,
            gt=float("-inf"), lt=float("inf"), ne=0,
        )
    self.effect_size = effect_size
    if isinstance(rel_effect_size, Sequence):
        for x in rel_effect_size:
            tea_tasting.utils.check_scalar(
                x, "rel_effect_size", typ=float | int,
                gt=float("-inf"), lt=float("inf"), ne=0,
            )
    elif rel_effect_size is not None:
        tea_tasting.utils.check_scalar(
            rel_effect_size, "rel_effect_size", typ=float | int,
            gt=float("-inf"), lt=float("inf"), ne=0,
        )
    self.rel_effect_size = rel_effect_size
    self.n_obs = (
        tea_tasting.utils.auto_check(n_obs, "n_obs")
        if n_obs is not None
        else tea_tasting.config.get_config("n_obs")
    )

aggr_cols: AggrCols property

Columns to be aggregated for a metric analysis.

analyze(data, control, treatment, variant=None)

Analyze a metric in an experiment.

Parameters:

Name	Type	Description	Default
data	IntoFrame | Table | dict[Any, Aggregates]	Experimental data.	required
control	Any	Control variant.	required
treatment	Any	Treatment variant.	required
variant	str | None	Variant column name.	None

Returns:

Type	Description
R	Analysis result.

Source code in src/tea_tasting/metrics/base.py

def analyze(
    self,
    data: narwhals.typing.IntoFrame | ibis.expr.types.Table | dict[
        Any, tea_tasting.aggr.Aggregates],
    control: Any,
    treatment: Any,
    variant: str | None = None,
) -> R:
    """Analyze a metric in an experiment.

    Args:
        data: Experimental data.
        control: Control variant.
        treatment: Treatment variant.
        variant: Variant column name.

    Returns:
        Analysis result.
    """
    aggr = aggregate_by_variants(
        data,
        aggr_cols=self.aggr_cols,
        variant=variant,
    )
    return self.analyze_aggregates(
        control=aggr[control],
        treatment=aggr[treatment],
    )

analyze_aggregates(control, treatment)

Analyze a metric in an experiment using aggregated statistics.

Parameters:

Name	Type	Description	Default
control	Aggregates	Control data.	required
treatment	Aggregates	Treatment data.	required

Returns:

Type	Description
MeanResult	Analysis result.

Source code in src/tea_tasting/metrics/mean.py

def analyze_aggregates(
    self,
    control: tea_tasting.aggr.Aggregates,
    treatment: tea_tasting.aggr.Aggregates,
) -> MeanResult:
    """Analyze a metric in an experiment using aggregated statistics.

    Args:
        control: Control data.
        treatment: Treatment data.

    Returns:
        Analysis result.
    """
    control = control.with_zero_div()
    treatment = treatment.with_zero_div()
    total = control + treatment
    covariate_coef = self._covariate_coef(total)
    covariate_mean = total.mean(self.numer_covariate) / total.mean(
        self.denom_covariate)
    return self._analyze_stats(
        contr_mean=self._metric_mean(control, covariate_coef, covariate_mean),
        contr_var=self._metric_var(control, covariate_coef),
        contr_count=control.count(),
        treat_mean=self._metric_mean(treatment, covariate_coef, covariate_mean),
        treat_var=self._metric_var(treatment, covariate_coef),
        treat_count=treatment.count(),
    )

solve_power(data, parameter='rel_effect_size')

Solve for a parameter of the power of a test.

Parameters:

Name	Type	Description	Default
data	IntoFrame | Table | Aggregates	Sample data.	required
parameter	Literal['power', 'effect_size', 'rel_effect_size', 'n_obs']	Parameter name.	'rel_effect_size'

Returns:

Type	Description
S	Power analysis result.

Source code in src/tea_tasting/metrics/base.py

def solve_power(
    self,
    data: (
        narwhals.typing.IntoFrame |
        ibis.expr.types.Table |
        tea_tasting.aggr.Aggregates
    ),
    parameter: Literal[
        "power", "effect_size", "rel_effect_size", "n_obs"] = "rel_effect_size",
) -> S:
    """Solve for a parameter of the power of a test.

    Args:
        data: Sample data.
        parameter: Parameter name.

    Returns:
        Power analysis result.
    """
    if not isinstance(data, tea_tasting.aggr.Aggregates):
        data = tea_tasting.aggr.read_aggregates(
            data=data,
            group_col=None,
            **self.aggr_cols._asdict(),
        )
    return self.solve_power_from_aggregates(data=data, parameter=parameter)

solve_power_from_aggregates(data, parameter='rel_effect_size')

Solve for a parameter of the power of a test.

Parameters:

Name	Type	Description	Default
data	Aggregates	Sample data.	required
parameter	Literal['power', 'effect_size', 'rel_effect_size', 'n_obs']	Parameter name.	'rel_effect_size'

Returns:

Type	Description
MeanPowerResults	Power analysis result.

Source code in src/tea_tasting/metrics/mean.py

def solve_power_from_aggregates(
    self,
    data: tea_tasting.aggr.Aggregates,
    parameter: Literal[
        "power", "effect_size", "rel_effect_size", "n_obs"] = "rel_effect_size",
) -> MeanPowerResults:
    """Solve for a parameter of the power of a test.

    Args:
        data: Sample data.
        parameter: Parameter name.

    Returns:
        Power analysis result.
    """
    tea_tasting.utils.check_scalar(
        parameter,
        "parameter",
        in_={"power", "effect_size", "rel_effect_size", "n_obs"},
    )

    data = data.with_zero_div()
    covariate_coef = self._covariate_coef(data)
    covariate_mean = data.mean(self.numer_covariate) / data.mean(
        self.denom_covariate)
    metric_mean = self._metric_mean(data, covariate_coef, covariate_mean)

    power, effect_size, rel_effect_size, n_obs = self._validate_power_parameters(
        metric_mean=metric_mean,
        sample_count=data.count(),
        parameter=parameter,
    )

    result = MeanPowerResults()
    for effect_size_i, rel_effect_size_i in zip(
        effect_size,
        rel_effect_size,
        strict=True,
    ):
        for n_obs_i in n_obs:
            parameter_value = self._solve_power_from_stats(
                sample_var=self._metric_var(data, covariate_coef),
                sample_count=n_obs_i,
                effect_size=effect_size_i,
                power=power,
            )
            result.append(MeanPowerResult(
                power=parameter_value if parameter == "power" else power,  # type: ignore
                effect_size=(
                    parameter_value
                    if parameter in {"effect_size", "rel_effect_size"}
                    else effect_size_i
                ),  # type: ignore
                rel_effect_size=(
                    parameter_value / metric_mean
                    if parameter in {"effect_size", "rel_effect_size"}
                    else rel_effect_size_i
                ),  # type: ignore
                n_obs=(
                    math.ceil(parameter_value)
                    if parameter == "n_obs"
                    else n_obs_i
                ),  # type: ignore
            ))

    return result