%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display
import beam
from beam.datasets import load_openproblems, load_openproblems_svg_featuresOpenProblems: batch integration and spatially variable genes
Provenance of the bundled tables
The raw single-cell data are not bundled. src/beam/data/openproblems_batch_integration.csv, openproblems_svg.csv and openproblems_svg_features.csv are reduced from the CC-BY-4.0 results JSON of the openproblems-bio/website repository at commit 76ce7f288da591b1b19c32cbfe8ce50bc3706ece, without the control and baseline methods. See src/beam/data/README.md. Reference: OpenProblems consortium (Nature Biotechnology 2025, DOI 10.1038/s41587-025-02694-w).
SHA=76ce7f288da591b1b19c32cbfe8ce50bc3706ece
BASE=https://raw.githubusercontent.com/openproblems-bio/website/$SHA/results
for task in batch_integration spatially_variable_genes; do
for f in results metric_info dataset_info method_info; do
curl -sf -o "$task.$f.json" "$BASE/$task/data/$f.json"
done
doneimport csv, json
# beam loads these exact file names (see beam.datasets._OP_TASKS).
OUTPUT = {
"batch_integration": "openproblems_batch_integration.csv",
"spatially_variable_genes": "openproblems_svg.csv",
}
def reduce_task(task):
results = json.load(open(f"{task}.results.json"))
method_info = json.load(open(f"{task}.method_info.json"))
baselines = {m["method_id"] for m in method_info if m["is_baseline"]}
rows = []
for r in results:
if r["method_id"] in baselines: # drop controls and baselines
continue
for metric_id, value in r["metric_values"].items():
score = "" if value in ("NA", None) else value # the source "NA" becomes empty
rows.append((r["method_id"], r["dataset_id"], metric_id, score))
rows.sort()
with open(OUTPUT[task], "w", newline="") as fh:
w = csv.writer(fh)
w.writerow(["method_id", "dataset_id", "metric_id", "score"])
w.writerows(rows)
# The spatial dataset features are parsed from the dataset id <source>/<technology>/<name>.
def svg_features():
datasets = sorted({r["dataset_id"] for r in json.load(open("spatially_variable_genes.results.json"))})
with open("openproblems_svg_features.csv", "w", newline="") as fh:
w = csv.writer(fh)
w.writerow(["dataset_id", "technology", "organism", "condition"])
for d in datasets:
_, technology, name = d.split("/")
organism = next((o for o in ("human", "mouse", "drosophila") if name.startswith(o)), "other")
condition = "cancer" if ("cancer" in name or "melanoma" in name) else "noncancer"
w.writerow([d, technology, organism, condition])The metric directions and reference DOIs on the cards come from the metric_info.json of each task. Every metric here is higher is better.
Batch integration
The task has 19 methods, 6 datasets and 13 scIB metrics. Coverage is uneven. The sparse hvg_overlap metric is dropped, and the methods with at least one dataset for every other metric are kept.
op = load_openproblems("batch_integration")
metrics = [m for m in op.metric_ids if m != "hvg_overlap"]
tensor = op.tensor(tuple(metrics))
keep = (~np.isnan(tensor).all(axis=1)).all(axis=1)
methods = [m for m, k in zip(op.method_names, keep) if k]
tensor = tensor[keep]
print(f"{len(methods)} methods, {len(metrics)} metrics, {len(op.dataset_names)} datasets")14 methods, 12 metrics, 6 datasets
card_data_consistency checks the scores against the 12 cards before normalization.
from beam.mcda import card_data_consistency, registry_context
ctx = registry_context(metrics, "saw")
pooled_native = np.nanmean(tensor, axis=1)
audit = card_data_consistency(
pooled_native, ctx.polarity, ctx.bounds,
baselines=ctx.baselines, targets=ctx.targets, noise_floors=ctx.noise_floors,
metric_ids=metrics,
)
print("scores consistent with the cards:", audit.ok)
for finding in audit.findings:
print(" ", finding.severity, finding.message)scores consistent with the cards: True
The run uses equal weights and TOPSIS.
scores = beam.Scores(
values=tensor,
tool_names=tuple(methods),
metric_ids=tuple(metrics),
dataset_names=op.dataset_names,
layout="long",
)
result = beam.rank(scores, weights="equal", method="topsis")
beam_order = np.argsort(result.result.ranks)
print("beam (equal weights, TOPSIS), top 5:")
for i in beam_order[:5]:
print(f" {result.tool_names[i]}")beam (equal weights, TOPSIS), top 5:
combat
harmonypy
harmony
scvi
scalex
The OpenProblems leaderboard is the mean of the per-metric scaled scores. The same rule on the raw scores:
mean_score = np.nanmean(np.nanmean(tensor, axis=1), axis=1)
mean_order = np.argsort(-mean_score)
print("mean of scores (OpenProblems-style), top 5:")
for i in mean_order[:5]:
print(f" {methods[i]}")
print()
print("beam top method:", result.tool_names[beam_order[0]])
print("mean-of-scores top method:", methods[mean_order[0]])mean of scores (OpenProblems-style), top 5:
scanvi
combat
scvi
harmonypy
harmony
beam top method: combat
mean-of-scores top method: scanvi
from beam import plot
# The outlined bar is the method the platform's mean-of-scores rule ranks first.
plot.ranking(result, ground_truth_tool=methods[mean_order[0]])Funky heatmap
The funky heatmap with three panels: the ARI marginal mean with its 95 percent mixed-effects interval, the leave-one-dataset-out rank span, and the SMAA rank-acceptability bar.
from beam.heterogeneity import mixed_effects_from_matrix, r_available
from beam.reporting import funky_heatmap_from_run
bio = {
"ari", "nmi", "asw_label", "isolated_label_f1", "isolated_label_asw",
"cell_cycle_conservation", "hvg_overlap", "clisi",
}
groups = ["bio" if m in bio else "batch" for m in result.metric_ids]
worth = worth_ci = None
if r_available():
ari_slice = op.tensor(("ari",))[keep, :, 0]
me = mixed_effects_from_matrix(ari_slice, methods, op.dataset_names)
by_name = {name: i for i, name in enumerate(me.method_names)}
worth = np.array(
[me.method_effects[by_name[m]] if m in by_name else np.nan for m in result.tool_names]
)
worth_ci = np.array(
[1.96 * me.method_effect_se[by_name[m]] if m in by_name else np.nan for m in result.tool_names]
)
funky = funky_heatmap_from_run(
result,
metric_groups=groups,
worth=worth,
worth_ci=worth_ci,
worth_label="ARI marginal mean\n(mixed-effects, 95% CI)",
show_aggregation_consensus=False,
title="OpenProblems batch integration: scores and rank robustness",
)
funkyMixed-effects on ARI
A mixed-effects model splits the ARI variance into a dataset effect and a residual. It needs R’s lme4.
from beam.heterogeneity import mixed_effects_from_matrix, r_available
ari = op.tensor(("ari",))[:, :, 0]
ari_keep = ~np.isnan(ari).all(axis=1)
ari_methods = [m for m, k in zip(op.method_names, ari_keep) if k]
ari = ari[ari_keep]
if r_available():
me = mixed_effects_from_matrix(ari, ari_methods, op.dataset_names)
print(f"dataset shift (ICC): {me.icc_dataset:.2f} of the ARI variance")
print(f"residual share: {me.residual_share:.2f}")
top = np.argsort(-me.method_effects)[:3]
print("highest ARI marginal means:")
for i in top:
print(f" {me.method_names[i]:14s} {me.method_effects[i]:.3f} +/- {me.method_effect_se[i]:.3f}")
else:
print("R with lme4 not available; skipping the mixed-effects fit.")
print("Provision it with envs/heterogeneity.yml.")dataset shift (ICC): 0.57 of the ARI variance
residual share: 0.43
highest ARI marginal means:
scanvi 0.679 +/- 0.066
bbknn 0.661 +/- 0.066
scvi 0.628 +/- 0.066
Metric validity
The scIB score weights biological conservation at 0.6 and batch correction at 0.4. beam.mcda.metric_validity correlates the metrics with Spearman over the method-by-dataset cells, oriented to higher-is-better (Campbell and Fiske 1959). See the validity explanation.
from beam.cards import polarities_for
from beam.mcda import metric_validity
validity = metric_validity(
tensor,
polarities_for(metrics),
groups,
metric_ids=list(metrics),
)
print("within-group agreement (convergent):")
for g, r in validity.convergent_by_group.items():
print(f" {g:6s} {r:.2f}")
print(f"between-group agreement (discriminant): {validity.mean_discriminant:.2f}")
print(f"convergent {validity.mean_convergent:.2f} > discriminant {validity.mean_discriminant:.2f}: {validity.discriminant_ok}")
print()
print("metrics that lean toward the other group:")
for name, group, within, between, nearest in validity.crossloading_metrics:
print(f" {name:20s} {group:6s} within {within:.2f} vs between {between:.2f}, leans {nearest}")within-group agreement (convergent):
bio 0.45
batch 0.24
between-group agreement (discriminant): 0.30
convergent 0.38 > discriminant 0.30: True
metrics that lean toward the other group:
asw_batch batch within 0.26 vs between 0.30, leans bio
asw_label bio within 0.36 vs between 0.38, leans batch
graph_connectivity batch within 0.34 vs between 0.59, leans bio
kbet batch within 0.18 vs between 0.31, leans bio
The mean correlation is 0.38 within groups and 0.30 between groups; 0.45 among the biological metrics and 0.24 among the batch metrics.
Metric reliability
beam.mcda.metric_reliability reports standardized Cronbach’s alpha per group from the same correlations (Cronbach 1951), with 0.7 as the usual cutoff. See the reliability explanation.
from beam.mcda import metric_reliability
reliability = metric_reliability(
tensor,
polarities_for(metrics),
groups,
metric_ids=list(metrics),
)
for g, alpha in reliability.alpha_by_group.items():
k = reliability.k_by_group[g]
r_bar = reliability.mean_inter_item_by_group[g]
print(f" {g:6s} alpha {alpha:.2f} over {k} metrics, mean inter-item r {r_bar:.2f}")
print()
print("groups below the 0.7 cutoff:", [g for g, _ in reliability.low_reliability_groups])
print("alpha if a batch metric is dropped:")
batch_alpha = reliability.alpha_by_group["batch"]
for name, group, alpha_without in reliability.alpha_if_dropped:
if group == "batch":
gain = "raises" if alpha_without > batch_alpha else "lowers"
print(f" drop {name:18s} alpha -> {alpha_without:.2f} ({gain})") bio alpha 0.85 over 7 metrics, mean inter-item r 0.45
batch alpha 0.62 over 5 metrics, mean inter-item r 0.24
groups below the 0.7 cutoff: ['batch']
alpha if a batch metric is dropped:
drop asw_batch alpha -> 0.55 (lowers)
drop graph_connectivity alpha -> 0.46 (lowers)
drop ilisi alpha -> 0.47 (lowers)
drop kbet alpha -> 0.62 (raises)
drop pcr alpha -> 0.67 (raises)
Alpha is 0.85 for the seven biological metrics and 0.62 for the five batch metrics. Removing pcr raises the batch alpha to 0.67.
Metric dimensionality
beam.mcda.metric_dimensionality counts the factors in each group with parallel analysis on the same correlations (Horn 1965, Glorfeld 1995). See the dimensionality explanation.
from beam.mcda import metric_dimensionality
dimensionality = metric_dimensionality(
tensor,
polarities_for(metrics),
groups,
metric_ids=list(metrics),
)
for g in dimensionality.k_by_group:
k = dimensionality.k_by_group[g]
pc1 = dimensionality.pc1_explained_by_group[g]
n_factors = dimensionality.parallel_components_by_group[g]
print(f" {g:6s} {n_factors} factor(s) over {k} metrics, first component explains {pc1:.2f}")
print()
print("read as one factor:", list(dimensionality.unidimensional_groups)) bio 2 factor(s) over 7 metrics, first component explains 0.54
batch 1 factor(s) over 5 metrics, first component explains 0.42
read as one factor: ['batch']
Parallel analysis keeps two factors for the biological group and one for the batch group.
Rank sensitivity
rank_sensitivity ranks under every combination of weighting, aggregation and dataset and splits the rank variance. COMET is left out: it is slow with 12 metrics.
from beam.mcda import rank_sensitivity
rs = rank_sensitivity(
tensor,
ctx.polarity,
methods=["saw", "topsis", "vikor", "promethee_ii"],
normalization=list(ctx.normalization),
bounds=list(ctx.bounds),
baselines=list(ctx.baselines),
targets=list(ctx.targets),
missing="worst",
tool_names=methods,
dataset_names=op.dataset_names,
)
print(f"{rs.n_combinations} combinations")
print(f" dataset: {rs.dataset_share:.3f} of the rank variance")
print(f" weighting: {rs.weighting_share:.3f}")
print(f" aggregation: {rs.aggregation_share:.3f}")
print(f" interactions: {rs.interaction_share:.3f}")
print(f" most influential factor: {rs.most_influential_factor}")96 combinations
dataset: 0.508 of the rank variance
weighting: 0.041
aggregation: 0.156
interactions: 0.295
most influential factor: dataset
The same split per method, with each method’s rank range:
plot.rank_sensitivity_by_tool(rs)specification_curve lists the rankings of the same grid.
from beam.mcda import specification_curve
curve = specification_curve(rs)
dom = curve.tool_names[curve.most_frequent_top_tool]
print(f"choices plus dataset ({curve.n_specifications} combinations): "
f"{dom} first in {curve.most_frequent_top_fraction * 100:.0f}%, "
f"{curve.n_distinct_top_tools} methods reach the top")
pooled = specification_curve(
rank_sensitivity(
result.matrix, ctx.polarity,
methods=["saw", "topsis", "vikor", "promethee_ii"],
normalization=list(ctx.normalization), bounds=list(ctx.bounds),
baselines=list(ctx.baselines), targets=list(ctx.targets),
missing="worst", tool_names=methods,
)
)
pdom = pooled.tool_names[pooled.most_frequent_top_tool]
print(f"choices only ({pooled.n_specifications} combinations): "
f"{pdom} first in {pooled.most_frequent_top_fraction * 100:.0f}%")
plot.specification_curve(curve)choices plus dataset (96 combinations): combat first in 35%, 9 methods reach the top
choices only (16 combinations): combat first in 56%
Dataset concordance
dataset_concordance ranks the methods within each dataset and compares every pair of orderings with Kendall tau-b.
conc = result.dataset_concordance
names = conc.dataset_names
print(f"mean agreement across datasets (Kendall tau-b): {conc.mean_pairwise_tau:.2f}")
print("least typical dataset:", names[conc.most_idiosyncratic_dataset])
print("mutually consistent groups:",
[tuple(names[d] for d in g) for g in conc.concordant_groups])
print("where methods depart most from their own average rank:")
for cell in conc.notable_cells[:5]:
side = "lower" if cell.deviation > 0 else "higher"
print(f" {conc.tool_names[cell.tool]} on {names[cell.dataset]}: "
f"rank {cell.rank}, {side} than its mean {cell.mean_rank:.1f}")mean agreement across datasets (Kendall tau-b): 0.49
least typical dataset: cellxgene_census/gtex_v9
mutually consistent groups: [('cellxgene_census/gtex_v9',), ('cellxgene_census/immune_cell_atlas',)]
where methods depart most from their own average rank:
scimilarity on cellxgene_census/gtex_v9: rank 12, lower than its mean 6.5
scimilarity on cellxgene_census/immune_cell_atlas: rank 1, higher than its mean 6.5
scanvi on cellxgene_census/gtex_v9: rank 7, lower than its mean 4.5
scanvi on cellxgene_census/immune_cell_atlas: rank 2, higher than its mean 4.5
combat on cellxgene_census/gtex_v9: rank 1, higher than its mean 2.5
plot.dataset_concordance(result)plot.dataset_struggle(result)Blind analysis
A blind analysis fixes the weighting, the aggregation and the metric set before the method names are known. beam.blind masks the names and shuffles the rows; beam.unblind restores them. The seal fingerprint is recorded in the manifest.
from beam import blind, unblind
blinded, seal = blind(scores, seed=0)
blind_run = beam.rank(blinded, weights="equal", method="topsis", seed=0, sensitivity=False)
restored = unblind(blind_run, seal)
named = beam.rank(scores, weights="equal", method="topsis", seed=0, sensitivity=False)
print("ranking identical after unblinding:",
dict(zip(named.tool_names, named.result.ranks))
== dict(zip(restored.tool_names, restored.result.ranks)))
print("top method after unblinding:", restored.top_tool)
print("blinding fingerprint:", blind_run.manifest["blinding"]["seal_sha256"][:12])ranking identical after unblinding: True
top method after unblinding: combat
blinding fingerprint: 14d2dba2852d
Pairwise superiority on ARI
pairwise_superiority compares the methods two at a time on ARI across the six datasets, with the ARI noise floor as the equivalence band.
from beam.mcda import pairwise_superiority
ari_kept = op.tensor(("ari",))[keep, :, 0]
ari_floor = registry_context(["ari"], "saw").noise_floors[0] or 0.0
sup = pairwise_superiority(ari_kept, "higher_is_better", rope=ari_floor, method_names=methods)
print(f"highest standing: {methods[sup.order[0]]} ({sup.standing[sup.order[0]]:.2f})")
print(f"pairs the sign test cannot separate: {len(sup.equivalent_pairs)} of {len(sup.per_pair)}")highest standing: scanvi (0.87)
pairs the sign test cannot separate: 73 of 91
With six datasets the sign test has little power.
pairwise_transitivity tests whether the pairwise majorities agree with one order.
from beam.mcda import pairwise_transitivity
trans = pairwise_transitivity(sup)
print(f"transitive: {trans.is_transitive}; circular triads: {trans.n_circular_triads} of {trans.n_triads}")transitive: False; circular triads: 1 of 364
The matrix orders the methods by the number of others outperformed. A red cell below the diagonal is a method outperforming one ranked above it.
plot.pairwise_majority(trans)bayesian_sign_comparison gives, for each pair, the posterior probability that one method is practically better.
from beam.mcda import bayesian_sign_comparison
bayes = bayesian_sign_comparison(sup)
decisive = sum(1 for p in bayes.per_pair if p.decision != "inconclusive")
print(f"pairs with a decisive posterior at 0.95: {decisive} of {len(bayes.per_pair)}")
plot.bayesian_comparison(bayes)pairs with a decisive posterior at 0.95: 18 of 91
Spatially variable genes
The task has 14 methods, 50 spatial datasets and one correlation metric, higher is better. The dataset features are the spatial assay technology, the organism, and a cancer or non-cancer condition, parsed from the dataset id.
svg = load_openproblems("spatially_variable_genes")
svg_features = load_openproblems_svg_features()
correlation = svg.tensor(("correlation",))[:, :, 0]
_, categorical = svg_features.aligned_to(svg.dataset_names)
print(f"{len(svg.method_names)} methods, {len(svg.dataset_names)} datasets")
from collections import Counter
print("technologies:", dict(Counter(categorical["technology"])))14 methods, 50 datasets
technologies: {'post_xenium': 2, 'visium': 20, 'dbitseq': 6, 'merfish': 5, 'seqfish': 1, 'slideseqv2': 5, 'starmap': 2, 'stereoseq': 5, 'slidetags': 4}
Mean correlation per method and assay technology:
import warnings
technologies = sorted(set(categorical["technology"]))
tech_array = np.array(categorical["technology"])
heat = np.full((len(svg.method_names), len(technologies)), np.nan)
for ti, tech in enumerate(technologies):
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
heat[:, ti] = np.nanmean(correlation[:, tech_array == tech], axis=1)
method_order = np.argsort(-np.nanmean(heat, axis=1))
plot.score_heatmap(
heat[method_order],
row_names=[svg.method_names[i] for i in method_order],
col_names=technologies,
row_label="spatially-variable-genes method (ordered by mean correlation)",
col_label="spatial assay technology",
value_label="mean correlation (higher is better)",
title="Mean correlation per method per assay (the column maximum moves between methods)",
)The Bradley-Terry tree takes each dataset as a set of pairwise method comparisons and splits the datasets by their features, with a ranking per leaf. It needs R’s psychotree.
from beam.heterogeneity import bradley_terry_tree, bttree_available
if bttree_available():
bt = bradley_terry_tree(
correlation,
svg.method_names,
svg.dataset_names,
categorical_features=categorical,
polarity="higher_is_better",
minsize=6,
)
print(f"split found: {bt.did_split}")
print(f"pooled ranking, top 3: {', '.join(bt.global_ranking()[:3])}")
print(bt.summary())
else:
bt = None
print("R with psychotree not available; skipping the Bradley-Terry tree.")
print("Provision it with envs/heterogeneity.yml.")split found: True
pooled ranking, top 3: spark_x, nnsvg, gpcounts
The Bradley-Terry tree splits the 50 datasets on organism, technology into 4 leaves; the global ranking is led by spark_x. The pooled recommendation does not hold everywhere: in a leaf of 10 datasets the ranking is led by nnsvg; in a leaf of 22 datasets the ranking is led by spanve.
Number of datasets and top method per leaf:
if bt is not None:
display(plot.bradley_terry_leaves(bt))Summary
On batch integration, the ranking with card-declared normalization and TOPSIS is compared with the mean of scores, and the biological and batch metric groups correlate at 0.30. On spatially variable genes, the Bradley-Terry tree gives a ranking per group of datasets.