%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from beam.cards import properties_for
from beam.mcda import (
aggregate_across_datasets,
critical_difference,
leave_one_metric_out,
run,
run_from_registry,
smaa,
smallest_weight_perturbation,
)
from beam.scenarios import (
all_scenarios,
normalization_failure_scenarios,
)
def polarity_for(scenario):
return [p.polarity for p in properties_for(list(scenario.metric_ids))]
def bounds_for(scenario):
props = properties_for(list(scenario.metric_ids))
return [(p.range_lower, p.range_upper) for p in props]
def unguarded_minmax(scenario):
"""Force plain min-max on every column, the behaviour the card defaults improve on."""
return run(
scenario.scores,
polarity_for(scenario),
normalization="min_max",
bounds=bounds_for(scenario),
metric_ids=list(scenario.metric_ids),
)
def print_section(label):
print()
print("=" * 64)
print(label)
print("=" * 64)Canonical simulated scenarios
Scenarios
- random: anti-correlated scores, so no method scores higher on both metrics. The top method depends on the weights.
- dominant: one method scores highest on every metric and is rank 1 under any weighting.
- ties: two methods have identical scores and share a rank under any weighting.
- odd dataset: one method ranks first on most datasets and another on one odd dataset. The cross-dataset rule on each metric card sets the overall order.
Set-up
The aggregation methods come from pymcdm, called on the matrix normalized by beam according to the metric cards. The weighting schemes are beam’s own.
Reports per scenario
For each scenario:
- Description and ground truth from the generator.
- Pooled tool by metric scores.
- Ranks under SAW with equal weights using
run_from_registry. - Leave-one-metric-out per-tool stability.
- SMAA rank acceptability over 500 weight samples.
- Triantaphyllou-Sanchez weight perturbation: the most fragile pair and whether the top rank is fragile.
for s in all_scenarios():
print_section(f"{s.name} ({s.kind})")
print(s.description)
print()
print("expected top-ranked method:",
"none" if s.expectation.expected_top_ranked is None
else s.method_names[s.expectation.expected_top_ranked])
if s.expectation.tied_pair is not None:
a, b = s.expectation.tied_pair
print("tied pair:", s.method_names[a], "==", s.method_names[b])
print()
print("scores:")
print(f" method {' '.join(f'{m:>9s}' for m in s.metric_ids)}")
for name, row in zip(s.method_names, s.scores):
print(f" {name:8s} {' '.join(f'{v:9.3f}' for v in row)}")
polarity = polarity_for(s)
base = run_from_registry(s.scores, s.metric_ids, weights="equal", method="saw")
print()
print("ranks (SAW + equal weights):")
for name, c, r in zip(s.method_names, base.composite, base.ranks):
print(f" {name:8s} composite={c:.3f} rank={int(r)}")
if s.scores.shape[1] >= 2:
loo = leave_one_metric_out(
s.scores, polarity,
metric_ids=list(s.metric_ids),
weights="equal", method="saw",
)
print()
print("leave-one-metric-out stability:")
for name, stab in zip(s.method_names, loo.rank_stability):
print(f" {name:8s} stability={stab:.2f}")
print(f" most influential metric: {s.metric_ids[loo.most_influential_metric]!r}")
sm = smaa(s.scores, polarity, n_samples=500, method="saw", seed=0)
print()
print("SMAA rank acceptability (rows = methods, columns = ranks 1..n):")
for name, row in zip(s.method_names, sm.rank_acceptability_index):
print(f" {name:8s} {[f'{v:.2f}' for v in row]}")
print("confidence factor:")
for name, c in zip(s.method_names, sm.confidence_factor):
print(f" {name:8s} {c:.2f}")
ts = smallest_weight_perturbation(s.scores, polarity, weights="equal", method="saw")
print()
print("Triantaphyllou-Sanchez weight perturbation:")
if ts.most_fragile_pair is not None:
p = ts.most_fragile_pair
print(
f" most fragile pair: {s.method_names[p.higher]} > {s.method_names[p.lower]} "
f"flips by delta={p.delta:+.3f} on {s.metric_ids[p.criterion]!r}"
)
else:
print(" no single-criterion weight change can flip any pair")
print(f" top rank is fragile (threshold 0.05): {ts.top_rank_is_fragile}")
================================================================
random (no_signal)
================================================================
ARI and runtime are drawn iid then re-paired so the two metrics are anti-correlated. No method Pareto-dominates the rest; the ranking depends entirely on the weighting. SMAA confidence is spread across the Pareto-frontier methods rather than concentrated on one.
expected top-ranked method: none
scores:
method ari runtime
m0 0.368 87.635
m1 0.185 69.337
m2 0.058 36.168
m3 0.070 45.279
m4 0.457 101.083
ranks (SAW + equal weights):
m0 composite=0.254 rank=4
m1 composite=0.276 rank=3
m2 composite=0.529 rank=1
m3 composite=0.426 rank=2
m4 composite=0.228 rank=5
leave-one-metric-out stability:
m0 stability=0.00
m1 stability=0.00
m2 stability=0.50
m3 stability=0.50
m4 stability=0.50
most influential metric: 'ari'
SMAA rank acceptability (rows = methods, columns = ranks 1..n):
m0 ['0.00', '0.52', '0.13', '0.35', '0.00']
m1 ['0.00', '0.00', '0.74', '0.11', '0.15']
m2 ['0.49', '0.00', '0.12', '0.22', '0.17']
m3 ['0.00', '0.46', '0.00', '0.26', '0.27']
m4 ['0.51', '0.02', '0.01', '0.06', '0.40']
confidence factor:
m0 0.00
m1 0.00
m2 0.49
m3 0.00
m4 0.51
Triantaphyllou-Sanchez weight perturbation:
most fragile pair: m2 > m0 flips by delta=-0.009 on 'runtime'
top rank is fragile (threshold 0.05): True
================================================================
dominant (dominant)
================================================================
Method 0 dominates on every metric. Under any positive weighting it ranks first; no single-criterion weight perturbation can move it off the top rank.
expected top-ranked method: m0
scores:
method ari runtime
m0 0.900 20.000
m1 0.131 155.756
m2 0.062 145.025
m3 0.055 104.245
m4 0.294 93.168
ranks (SAW + equal weights):
m0 composite=0.950 rank=1
m1 composite=0.065 rank=4
m2 composite=0.049 rank=5
m3 composite=0.125 rank=3
m4 composite=0.272 rank=2
leave-one-metric-out stability:
m0 stability=1.00
m1 stability=0.00
m2 stability=0.00
m3 stability=0.50
m4 stability=1.00
most influential metric: 'runtime'
SMAA rank acceptability (rows = methods, columns = ranks 1..n):
m0 ['1.00', '0.00', '0.00', '0.00', '0.00']
m1 ['0.00', '0.00', '0.18', '0.33', '0.48']
m2 ['0.00', '0.00', '0.00', '0.51', '0.49']
m3 ['0.00', '0.00', '0.82', '0.16', '0.03']
m4 ['0.00', '1.00', '0.00', '0.00', '0.00']
confidence factor:
m0 1.00
m1 0.00
m2 0.00
m3 0.00
m4 0.00
Triantaphyllou-Sanchez weight perturbation:
most fragile pair: m1 > m2 flips by delta=-0.013 on 'ari'
top rank is fragile (threshold 0.05): False
================================================================
ties (ties)
================================================================
Methods 0 and 1 have identical scores, set above the rest on both metrics, so they tie for the top rank under any weighting.
expected top-ranked method: none
tied pair: m0 == m1
scores:
method ari runtime
m0 0.900 15.000
m1 0.900 15.000
m2 0.070 87.635
m3 0.058 45.279
m4 0.457 36.168
ranks (SAW + equal weights):
m0 composite=0.950 rank=1
m1 composite=0.950 rank=1
m2 composite=0.035 rank=5
m3 composite=0.216 rank=4
m4 composite=0.479 rank=3
leave-one-metric-out stability:
m0 stability=1.00
m1 stability=1.00
m2 stability=0.50
m3 stability=0.50
m4 stability=1.00
most influential metric: 'runtime'
SMAA rank acceptability (rows = methods, columns = ranks 1..n):
m0 ['1.00', '0.00', '0.00', '0.00', '0.00']
m1 ['1.00', '0.00', '0.00', '0.00', '0.00']
m2 ['0.00', '0.00', '0.00', '0.02', '0.98']
m3 ['0.00', '0.00', '0.00', '0.98', '0.02']
m4 ['0.00', '0.00', '1.00', '0.00', '0.00']
confidence factor:
m0 1.00
m1 1.00
m2 0.00
m3 0.00
m4 0.00
Triantaphyllou-Sanchez weight perturbation:
most fragile pair: m3 > m2 flips by delta=-0.488 on 'runtime'
top rank is fragile (threshold 0.05): False
================================================================
odd_dataset (odd_dataset)
================================================================
Method 0 is best on 4 out of 5 datasets; method 1 is best on the last (odd) dataset. After cross-dataset aggregation with the per-metric recommended rule, the pooled ranking still has method 0 first.
expected top-ranked method: m0
scores:
method ari runtime
m0 0.608 38.622
m1 0.322 60.108
m2 0.149 91.515
m3 0.173 84.922
ranks (SAW + equal weights):
m0 composite=0.804 rank=1
m1 composite=0.404 rank=2
m2 composite=0.075 rank=4
m3 composite=0.130 rank=3
leave-one-metric-out stability:
m0 stability=1.00
m1 stability=1.00
m2 stability=1.00
m3 stability=1.00
most influential metric: 'ari'
SMAA rank acceptability (rows = methods, columns = ranks 1..n):
m0 ['1.00', '0.00', '0.00', '0.00']
m1 ['0.00', '1.00', '0.00', '0.00']
m2 ['0.00', '0.00', '0.00', '1.00']
m3 ['0.00', '0.00', '1.00', '0.00']
confidence factor:
m0 1.00
m1 0.00
m2 0.00
m3 0.00
Triantaphyllou-Sanchez weight perturbation:
no single-criterion weight change can flip any pair
top rank is fragile (threshold 0.05): False
SMAA per scenario
Fraction of weight samples with each method ranked first. Gold is the method built to rank first and green a tied pair built as such.
scenarios = all_scenarios()
n_scen = len(scenarios)
fig, axes = plt.subplots(1, n_scen, figsize=(3.0 * n_scen, 3.2), sharey=True)
for ax, s in zip(axes, scenarios):
pol = polarity_for(s)
rep = smaa(s.scores, pol, n_samples=500, method="saw", seed=0)
top = s.expectation.expected_top_ranked
tied = s.expectation.tied_pair
colors = []
for i in range(len(s.method_names)):
if top is not None and i == top:
colors.append("#d9a521")
elif tied is not None and i in tied:
colors.append("#55a868")
else:
colors.append("#3a7ca5")
ax.bar(range(len(s.method_names)), rep.confidence_factor, color=colors)
if top is not None:
truth = f"truth: {s.method_names[top]} ranks first"
elif tied is not None:
truth = f"truth: tie {s.method_names[tied[0]]}=={s.method_names[tied[1]]}"
else:
truth = "truth: no single top-ranked method"
ax.set_title(f"{s.name}\n({s.kind})\n{truth}")
ax.set_xticks(range(len(s.method_names)))
ax.set_xticklabels(s.method_names, rotation=45, ha="right")
ax.set_ylim(0, 1.05)
ax.set_xlabel("method")
ax.set_ylabel("SMAA confidence factor")
fig.tight_layout()
plt.show()Funky heatmap
The funky heatmap of the random scenario, with the rank span across the five aggregations and the SMAA rank-acceptability bar. The scenarios have one dataset, so there is no leave-one-dataset-out span.
import beam
from beam.scenarios import random_scenario
s = random_scenario()
funky_run = beam.rank(s.scores, metric_ids=s.metric_ids, tool_names=s.method_names)
from beam.reporting import funky_heatmap_from_run
funky_heatmap_from_run(funky_run, title="Random scenario: scores and rank robustness")Ranks per dataset in the odd-dataset scenario
odd = next(s for s in scenarios if s.kind == "odd_dataset")
tensor = odd.scores_per_dataset
n_datasets = tensor.shape[1]
ari_per_dataset_ranks = np.zeros((len(odd.method_names), n_datasets), dtype=int)
for d in range(n_datasets):
col = tensor[:, d, 0]
order = np.argsort(-col, kind="stable")
for rank_pos, tool_idx in enumerate(order, start=1):
ari_per_dataset_ranks[tool_idx, d] = rank_pos
from beam import plot
truth_top = odd.method_names[odd.expectation.expected_top_ranked]
plot.rank_heatmap(
ari_per_dataset_ranks,
row_names=odd.method_names,
col_names=[f"d{i}" for i in range(n_datasets)],
row_label="method",
col_label="dataset",
title=f"truth: {truth_top} ranks first overall; one odd dataset reorders locally",
)Dataset concordance
dataset_concordance ranks the methods within each dataset and reports the Kendall tau-b agreement between every pair, and the dataset least in agreement with the rest.
odd_scores = beam.Scores(
values=odd.scores_per_dataset,
tool_names=odd.method_names,
metric_ids=odd.metric_ids,
dataset_names=tuple(f"d{i}" for i in range(n_datasets)),
layout="long",
)
odd_run = beam.rank(odd_scores, weights="equal", method="saw", missing="worst")
conc = odd_run.dataset_concordance
names = conc.dataset_names
print(f"mean agreement across datasets (Kendall tau-b): {conc.mean_pairwise_tau:.2f}")
print("least typical dataset:", names[conc.most_idiosyncratic_dataset])
print("mutually consistent groups:",
[tuple(names[d] for d in g) for g in conc.concordant_groups])mean agreement across datasets (Kendall tau-b): 0.47
least typical dataset: d4
mutually consistent groups: [('d0', 'd1', 'd2', 'd3'), ('d4',)]
plot.dataset_concordance(odd_run)Rank of each method per dataset minus its mean rank:
plot.dataset_struggle(odd_run)Normalization strategy
In two scenarios the top method under min-max for every metric differs from the one under the card defaults. See normalization and scales.
for s in normalization_failure_scenarios():
print_section(f"{s.name} ({s.kind})")
print(s.description)
print()
unguarded = unguarded_minmax(s)
guarded = run_from_registry(s.scores, s.metric_ids, weights="equal", method="saw")
print("ranks: unguarded all-min_max vs card defaults")
print(f" method {'min_max':>10s} {'card':>10s}")
for name, ru, rg in zip(s.method_names, unguarded.ranks, guarded.ranks):
flag = " <- flips" if ru != rg else ""
print(f" {name:8s} {int(ru):>10d} {int(rg):>10d}{flag}")
print()
print("card normalization strategy per metric:")
for mid, strat in zip(s.metric_ids, guarded.normalization):
print(f" {mid:10s} {strat}")
if unguarded.warnings:
print()
print("guard warnings raised under unguarded min-max:")
for w in unguarded.warnings:
print(f" - {w}")
================================================================
outlier_runtime (minmax_heavy_tail)
================================================================
Five methods, metrics ari and runtime. Methods m0 to m3 sit on a 10, 20, 40, 80 second ladder; m4 is a 5000 second outlier. Under unguarded all-min_max the outlier compresses the good methods to near 0.99 on runtime, so a tiny ARI difference decides the order and m1 comes out on top. Under the card default log_min_max the runtime ladder survives and m0 ranks first.
ranks: unguarded all-min_max vs card defaults
method min_max card
m0 2 1 <- flips
m1 1 2 <- flips
m2 3 3
m3 4 4
m4 5 5
card normalization strategy per metric:
ari baseline_relative
runtime log_min_max
guard warnings raised under unguarded min-max:
- metric runtime: min_max used an empirical upper bound; normalized values are not comparable across different method sets
- metric runtime: min_max on a heavy-tailed column (max/median 125); one outlier compresses the rest. Consider log_min_max or rank.
================================================================
chance_baseline (minmax_chance_baseline)
================================================================
Three methods, metrics ari and runtime. m0 is at chance (ARI 0), m1 is modestly better (ARI 0.20) but slower, m2 is strong overall. Unguarded min-max scores the chance ARI as 0.5 and ranks m0 above m1. The card default baseline_relative scores chance as 0 and ranks m1 above m0. m2 ranks first either way.
ranks: unguarded all-min_max vs card defaults
method min_max card
m0 2 3 <- flips
m1 3 2 <- flips
m2 1 1
card normalization strategy per metric:
ari baseline_relative
runtime log_min_max
guard warnings raised under unguarded min-max:
- metric runtime: min_max used an empirical upper bound; normalized values are not comparable across different method sets
Normalized runtime under min-max and under the card default log_min_max, in the scenario with one slow method:
outlier = next(s for s in normalization_failure_scenarios() if s.kind == "minmax_heavy_tail")
runtime_col = list(outlier.metric_ids).index("runtime")
mm = unguarded_minmax(outlier).normalized[:, runtime_col]
lg = run_from_registry(outlier.scores, outlier.metric_ids).normalized[:, runtime_col]
x = np.arange(len(outlier.method_names))
width = 0.38
fig, ax = plt.subplots(figsize=(5.5, 3.2))
ax.bar(x - width / 2, mm, width, label="min_max", color="#c44e52")
ax.bar(x + width / 2, lg, width, label="log_min_max", color="#3a7ca5")
ax.set_xticks(x)
ax.set_xticklabels(outlier.method_names)
ax.set_xlabel("method")
ax.set_ylabel("normalized runtime (higher = faster)")
ax.set_title("Runtime outlier crushes the min-max scale")
# overlay the ground truth: the actual runtime in seconds, on a log scale
true_runtime = outlier.scores[:, runtime_col]
ax2 = ax.twinx()
ax2.plot(x, true_runtime, "k--o", label="true runtime (s)")
ax2.set_yscale("log")
ax2.set_ylabel("true runtime (s, log scale)")
lines1, labels1 = ax.get_legend_handles_labels()
lines2, labels2 = ax2.get_legend_handles_labels()
ax.legend(lines1 + lines2, labels1 + labels2, loc="center right")
fig.tight_layout()
plt.show()The dashed line is the runtime in seconds. Under min-max the four fast methods (m0 to m3) have almost the same value.
Critical difference
The Demsar (2006) Friedman test and Nemenyi post-hoc on the ARI per dataset of the odd-dataset scenario. See critical difference.
odd = next(s for s in scenarios if s.kind == "odd_dataset")
ari_by_dataset = odd.scores_per_dataset[:, :, 0] # tool by dataset, higher is better
cd = critical_difference(ari_by_dataset, higher_is_better=True, tool_names=odd.method_names)
print(f"Friedman statistic = {cd.friedman_statistic:.3f}, p = {cd.friedman_pvalue:.3f}")
print(f"critical difference (alpha={cd.alpha}) = {cd.critical_difference:.3f}")
print("average ranks (1 = best):")
for i in cd.order:
print(f" {odd.method_names[i]:8s} {cd.average_ranks[i]:.2f}")
groups = [tuple(odd.method_names[i] for i in c) for c in cd.cliques]
print("cliques (not significantly different):", groups or "none")Friedman statistic = 5.400, p = 0.145
critical difference (alpha=0.05) = 2.098
average ranks (1 = best):
m0 1.60
m1 2.20
m3 2.80
m2 3.40
cliques (not significantly different): [('m0', 'm1', 'm3', 'm2')]
plot.critical_difference_band(cd)With five datasets the Friedman test does not reject at alpha 0.05.
pairwise_superiority counts how often each method outperforms another across the five datasets.
from beam.mcda import pairwise_superiority
sup = pairwise_superiority(ari_by_dataset, "higher_is_better", method_names=odd.method_names)
print(f"highest standing: {odd.method_names[sup.order[0]]} ({sup.standing[sup.order[0]]:.2f})")
print(f"pairs the sign test cannot separate: {len(sup.equivalent_pairs)} of {len(sup.per_pair)}")highest standing: m0 (0.80)
pairs the sign test cannot separate: 6 of 6
pairwise_transitivity tests whether the pairwise majorities agree with one order.
from beam.mcda import pairwise_transitivity
trans = pairwise_transitivity(sup)
print(f"transitive: {trans.is_transitive}; circular triads: {trans.n_circular_triads} of {trans.n_triads}")transitive: True; circular triads: 0 of 4
The matrix orders the methods by the number of others outperformed. A red cell below the diagonal is a method outperforming one ranked above it.
plot.pairwise_majority(trans)bayesian_sign_comparison gives the posterior probability that one method is practically better than another.
from beam.mcda import bayesian_sign_comparison
bayes = bayesian_sign_comparison(sup)
decisive = sum(1 for p in bayes.per_pair if p.decision != "inconclusive")
print(f"pairs with a decisive posterior at 0.95: {decisive} of {len(bayes.per_pair)}")
plot.bayesian_comparison(bayes)pairs with a decisive posterior at 0.95: 1 of 6
Heterogeneity on a synthetic interaction
A synthetic method by dataset by metric case with a ranking reversal: method A ranks first on the “easy” datasets and method C on the “hard” ones, with method B in between. The regime feature marks the group of each dataset.
rng = np.random.default_rng(0)
het_methods = ["method_A", "method_B", "method_C"]
het_datasets = [f"easy_{i}" for i in range(4)] + [f"hard_{i}" for i in range(4)]
regime = ["easy"] * 4 + ["hard"] * 4
het = np.empty((3, 8))
for j in range(8):
means = [0.9, 0.6, 0.4] if regime[j] == "easy" else [0.4, 0.6, 0.9]
het[:, j] = np.clip(np.array(means) + rng.normal(0, 0.05, 3), 0, 1)A mixed-effects model on this metric. With one observation per cell the residual is the method-by-dataset interaction plus noise. It needs R.
from beam.heterogeneity import (
bradley_terry_tree,
bttree_available,
mixed_effects_from_matrix,
r_available,
)
if r_available():
me = mixed_effects_from_matrix(het, het_methods, het_datasets)
print(f"dataset shift (ICC): {me.icc_dataset:.2f}")
print(f"residual share: {me.residual_share:.2f} (interaction upper bound)")
else:
print("R with lme4 not available; skipping the mixed-effects fit.")dataset shift (ICC): 0.00
residual share: 1.00 (interaction upper bound)
The Bradley-Terry tree splits the datasets on regime, with a different top method per leaf.
if bttree_available():
bt = bradley_terry_tree(
het,
het_methods,
het_datasets,
categorical_features={"regime": regime},
polarity="higher_is_better",
minsize=3,
)
print(f"split found: {bt.did_split}")
print(bt.summary())
else:
print("R with psychotree not available; skipping the Bradley-Terry tree.")split found: False
The Bradley-Terry tree found no dataset feature that splits the method ranking at alpha 0.05 over 8 datasets, so the ranking is reported as one Bradley-Terry model over all of them, led by method_A. With this many datasets the split test has few observations to work with, the same small-sample limit the critical-difference diagram shows; a benchmark with more datasets is where a split can appear.
Card consistency
card_data_consistency checks the raw scores against the cards. An ARI of 1.5, outside the declared range of [-1, 1], is flagged:
from beam.mcda import card_data_consistency
scenarios_by_name = {s.name: s for s in all_scenarios()}
dom = scenarios_by_name["dominant"]
props = properties_for(list(dom.metric_ids))
polarity = [p.polarity for p in props]
bounds = [(p.range_lower, p.range_upper) for p in props]
clean = card_data_consistency(dom.scores, polarity, bounds, metric_ids=list(dom.metric_ids))
print("scenario as built, consistent with the cards:", clean.ok)
corrupted = dom.scores.copy()
corrupted[0, 0] = 1.5
flagged = card_data_consistency(corrupted, polarity, bounds, metric_ids=list(dom.metric_ids))
print("after injecting ARI = 1.5, consistent:", flagged.ok)
for finding in flagged.violations:
print(" ", finding.severity, finding.message)scenario as built, consistent with the cards: True
after injecting ARI = 1.5, consistent: False
violation ari: 1 of 5 observed scores outside the declared range, 1 above 1 (max 1.5)
Rank sensitivity with one dataset
With one dataset rank_sensitivity has two factors, the weighting and the aggregation. In the dominant scenario one method is rank 1 in every combination and has no rank variance. In the random scenario the order depends on both.
from beam.mcda import rank_sensitivity, specification_curve
for name in ["dominant", "random"]:
s = scenarios_by_name[name]
pol = [p.polarity for p in properties_for(list(s.metric_ids))]
rs = rank_sensitivity(s.scores, pol, tool_names=s.method_names)
curve = specification_curve(rs)
top = curve.tool_names[curve.most_frequent_top_tool]
print(f"{name:9s}: dataset_share={rs.dataset_share} "
f"weighting={rs.weighting_share:.2f} aggregation={rs.aggregation_share:.2f} "
f"interaction={rs.interaction_share:.2f}")
print(f" {curve.n_specifications} specifications; top method {top} "
f"first in {curve.most_frequent_top_fraction * 100:.0f}%, "
f"{curve.n_distinct_top_tools} method(s) reach the top")dominant : dataset_share=None weighting=0.09 aggregation=0.67 interaction=0.24
20 specifications; top method m0 first in 100%, 1 method(s) reach the top
random : dataset_share=None weighting=0.34 aggregation=0.46 interaction=0.20
20 specifications; top method m4 first in 60%, 3 method(s) reach the top
specification_curve reports the dominant method first in every combination.
The rank variance split per method; a method with a fixed rank has none.
for name in ["dominant", "random"]:
s = scenarios_by_name[name]
pol = [p.polarity for p in properties_for(list(s.metric_ids))]
rs = rank_sensitivity(s.scores, pol, tool_names=s.method_names)
display(plot.rank_sensitivity_by_tool(rs, title=f"{name}: rank variance by factor and method"))A blind analysis fixes the weighting and the aggregation before the method names are known. beam.blind masks the names; beam.unblind restores them.
from beam import blind, unblind
s = scenarios_by_name["random"]
scenario_scores = beam.Scores(
values=s.scores,
tool_names=tuple(s.method_names),
metric_ids=tuple(s.metric_ids),
dataset_names=None,
layout="wide",
)
blinded, seal = blind(scenario_scores, seed=0)
blind_run = beam.rank(blinded, weights="equal", method="saw", seed=0, sensitivity=False)
restored = unblind(blind_run, seal)
named = beam.rank(scenario_scores, weights="equal", method="saw", seed=0, sensitivity=False)
print("ranking identical after unblinding:",
dict(zip(named.tool_names, named.result.ranks))
== dict(zip(restored.tool_names, restored.result.ranks)))
print("blinding fingerprint:", blind_run.manifest["blinding"]["seal_sha256"][:12])ranking identical after unblinding: True
blinding fingerprint: 5761a41d5107