Leaderboard Explore Compare Profile Community Blog FAQ Competition 2024 Report Issue

Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021rotation-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.646
2	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.614
3	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.606
4	resnet-152_v2_pytorch	.589
5	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.581
6	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.571
7	fixres_resnext101_32x48d_wsl	.568
8	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.565
9	resnet_152_v2	.561
10	vit_large_patch14_clip_336:laion2b_ft_in1k	.559
11	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.546
12	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.539
13	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.528
14	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.527
15	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.516
16	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.506
17	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.500
18	effnetv2m_custom384	.489
19	vit_large_patch14_clip_224:openai_ft_in1k	.488
20	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.478
21	convnext_base:clip_laiona_augreg_ft_in1k_384	.476
22	convnext_large:fb_in22k_ft_in1k	.467
23	vit_large_patch14_clip_224:laion2b_ft_in1k	.457
24	pnasnet_large	.431
25	vit_base_patch16_clip_224:openai_ft_in1k	.429
26	custom_model_cv_18_dagger_408	.425
27	convnext_xlarge:fb_in22k_ft_in1k	.417
28	resnet_101_v2	.416
29	resnet_50_v2	.413
30	resnext101_32x48d_wsl	.405
31	CLIP_resnet50	.404
32	CLIP_resnet50_float32	.404
33	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.404
34	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.400
35	resnext101_32x16d_wsl	.391
36	resnext101_32x32d_wsl	.385
37	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.380
38	SWSL_resnet50	.370
39	SWSL_resnext101_32x8d	.368
40	CLIP-RN50	.367
41	pnasnet_large-tf	.357
42	effnetb1_272x240	.356
43	resnext101_32x8d_wsl	.351
44	nasnet_large	.344
45	cvt_cvt-13-384-in1k_4_LucyV4	.343
46	cvt_cvt-21-384-in1k_4_LucyV4	.338
47	convnext_tiny:in12k_ft_in1k	.334
48	nasnet_large-tf	.325
49	vision_transformer_vit_large_patch16_224	.308
50	antialias-resnet152	.282
51	inception_v3	.273
52	CLIP_ViT-B_32	.273
53	focalnet_tiny_lrf_in1k	.273
54	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.268
55	inception_v4	.266
56	inception_v3-tf	.264
57	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.261
58	resnet152_imagenet_full	.259
59	convnext_femto_ols:d1_in1k	.255
60	resnet101	.255
61	resnet50-VITO-8deg-cc	.254
62	resnet101_imagenet_full	.250
63	vit_relpos_base_patch16_clsgap_224:sw_in1k	.249
64	resnet-152_v2-tf	.239
65	convnext_large_imagenet_full_seed-0	.236
66	resnet_152_v1	.236
67	cv_18_dagger_408_pretrained	.226
68	xception-keras	.225
69	antialiased-rnext101_32x8d	.225
70	convnext_tiny_imagenet_full_seed-0	.224
71	resnet50_random_l2_perturb	.223
72	tv_efficientnet-b1	.221
73	cvt_cvt-13-224-in1k_4_LucyV4	.218
74	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.213
75	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.212
76	convnext_base_imagenet_full_seed-0	.206
77	resnet50	.206
78	inception_v4-tf	.204
79	resnet50_imagenet_full	.204
80	deit_base_imagenet_full_seed-0	.201
81	convnext_small_imagenet_full_seed-0	.197
82	deit_large_imagenet_full_seed-0	.196
83	resnet_101_v1	.195
84	antialiased-r50	.195
85	grcnn_109	.194
86	resnet50-moclr8deg	.190
87	deit_small_imagenet_full_seed-0	.190
88	ViT_B_16_imagenet1k	.182
89	cvt_cvt-21-224-in1k_4_LucyV4	.181
90	densenet-201-keras	.181
91	xception	.179
92	Res2Net50_26w_4s	.177
93	pnasnet_large_pytorch	.175
94	densenet-201	.172
95	densenet_201_pytorch	.172
96	resnet_50_v1_spiking	.172
97	efficientnet_b2_imagenet_full	.167
98	vgg_19	.166
99	efficientnet_b0	.165
100	densenet-121-keras	.164
101	convnext_tiny_sup	.161
102	ViT_L_32_imagenet1k	.161
103	grcnn	.161
104	resnet34_imagenet_full	.160
105	resnet50_ewconplus_wzc	.160
106	resnet-50_v2-tf	.156
107	densenet-121	.155
108	resnet50_tutorial	.154
109	resnet_50_v1	.154
110	resnet-50-pytorch	.154
111	resnet50-sup	.154
112	mobilenet_v2_1.4_224-tf	.153
113	resnet50_random_linf8_perturb	.152
114	ViT_B_32_imagenet1k	.152
115	AdvProp_efficientnet-b6	.150
116	resnet50-SIN_IN_IN	.150
117	resnet_SIN_IN_FT_IN	.150
118	BiT-S-R50x3	.149
119	inception_v2-tf	.149
120	resnet-101_v1-tf	.149
121	resnet50_byol	.144
122	efficientnet-b7	.143
123	nasnet_mobile	.143
124	mobilenet_v2_1_0_224	.140
125	AT_efficientnet-b2	.137
126	AdvProp_efficientnet-b2	.137
127	resnet50-vitoimagevidnet8	.135
128	cornet_s	.133
129	BiT-S-R152x2	.132
130	densenet-169-keras	.131
131	voneresnet-50-non_stochastic	.130
132	BiT-S-R101x3	.130
133	mobilenet_v1_0.75_224	.130
134	mobilenet_v2_1.0_224-tf	.130
135	resnet18_imagenet_full	.130
136	efficientnet_b0_imagenet_full	.129
137	mobilenet_v2_1_3_224	.124
138	ReAlnet04	.124
139	AdvProp_efficientnet-b7	.123
140	inception_v1	.123
141	nasnet_mobile-tf	.121
142	ReAlnet03	.120
143	resnet50-SIN_IN	.119
144	resnet-50_v1-tf	.118
145	convnext_small_imagenet_100_seed-0	.117
146	ReAlnet10	.116
147	resnet_50_v1_spiking_l4a	.116
148	vgg_16	.116
149	AdvProp_efficientnet-b4	.115
150	AT_efficientnet-b4	.115
151	ReAlnet06	.115
152	ReAlnet01	.114
153	ReAlnet02	.114
154	ReAlnet09	.114
155	resnet-152_v1-tf	.114
156	densenet-169	.112
157	mobilenet_v2_1.3_224-tf	.109
158	resnet50_moco_v2	.105
159	BiT-S-R152x4	.105
160	efficientnet_b1_imagenet_full	.101
161	mobilenet_v1_1.0_192	.100
162	inception_v1-tf	.098
163	ReAlnet05	.098
164	mobilenet_v2_1.0_160-tf	.096
165	vonegrcnn_47e	.096
166	ReAlnet07	.095
167	mobilenet_v2_1_0_192	.094
168	CORnet-S	.094
169	mobilenet_v1_1.0_224	.092
170	ReAlnet08	.092
171	mobilenet_v2_1_0_160	.091
172	mobilenet_v2_0_5_224	.091
173	resnet-34	.090
174	mobilenet_v2_0_75_192	.089
175	BiT-S-R50x1	.089
176	mobilenet_v2_0_75_160	.088
177	mobilenet_v2_1-4_224_pytorch	.087
178	mobilenet_v2_1_4_224	.087
179	resnet50_linf_4_robust	.085
180	squeezenet1_1	.085
181	BiT-S-R101x1	.085
182	vonegrcnn_62e_nobn	.085
183	regnet_y_400mf	.085
184	vgg-16-keras	.084
185	vonegrcnn_47e	.081
186	resnet50_simclr	.081
187	mobilenet_v2_0_75_224	.080
188	vonegrcnn_52e_full	.080
189	resnet-18	.079
190	vonegrcnn_52e_full	.079
191	mobilenet_v2_0_5_192	.079
192	AdvProp_efficientnet-b0	.078
193	vonegrcnn_62e_nobn	.075
194	shufflenet_v2_x1_0	.074
195	resnet50_l2_3_robust	.073
196	mobilenet_v1_0.5_192	.073
197	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.073
198	0.5x_resnet-18_LC_w_sh_1_iter	.070
199	mobilenet_v2_0.75_160-tf	.070
200	alexnet	.069
201	mobilenet_v2_1.0_192-tf	.069
202	texture_shape_alexnet_trained_on_SIN	.068
203	mobilenet_v2_0.75_224-tf	.067
204	mobilenet_v2_0.75_192-tf	.064
205	texture_shape_resnet50_trained_on_SIN	.064
206	mobilenet_v2_0.35_192	.064
207	mobilenet_v2_1_0_128	.063
208	mobilenet_v2_0.75_96	.063
209	resnet50_eMMCR_Vanilla	.063
210	mobilenet_v1_0.25_224	.062
211	squeezenet1_0	.062
212	resnet50_robust_l2_eps1	.062
213	resnet50_linf_8_robust	.061
214	mobilenet_v2_0.5_192-tf	.061
215	mobilenet_v1_1.0_128	.061
216	AdvProp_efficientnet-b8	.061
217	mobilenet_v1_0.25_192	.060
218	mobilenet_v1_0.75_160	.060
219	mobilenet_v2_0.35_224	.059
220	blt_vs	.059
221	konkle_alexnetgn_ipcl_ref01_primary_model	.059
222	mobilenet_v1_0.5_224	.058
223	resnet50_imagenet_100_seed-0	.058
224	alexnet_random_linf8_perturb	.058
225	resnet-50-robust	.058
226	imagenet_l2_3_0	.058
227	resnet50_robust_l2_eps3	.058
228	voneresnet-50	.057
229	vgg-19-keras	.057
230	alexnet_reduced_aliasing_early_checkpoint	.056
231	mobilenet_v2_0.5_224-tf	.055
232	mobilenet_v2_0.75_128	.055
233	mobilenet_v2_0.35_96	.055
234	resnet50-SIN	.055
235	convnext_small_imagenet_1_seed-0	.054
236	mobilenet_v2_1.0_128-tf	.054
237	mobilenet_v1_0.75_128	.053
238	AlexNet_SIN	.052
239	mobilenet_v2_0.35_160	.052
240	mobilenet_v1_0.75_192	.052
241	voneresnet-50-robust	.052
242	mobilenet_v1_1.0_160	.050
243	alexnet_random_l2_3_perturb	.050
244	mobilenet_v2_0.5_96	.050
245	mobilenet_v2_0.5_160	.049
246	alexnet_ks_torevert	.049
247	alexnet-baseline	.049
248	alexnet	.049
249	alexnet	.049
250	alexnet	.049
251	alexnet	.049
252	alexnet	.049
253	alexnet	.049
254	alexnet	.049
255	alexnet	.049
256	alexnet	.049
257	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.049
258	mobilenet_v2_0.35_128	.048
259	mobilenet_v1_0.5_128	.046
260	vonealexnet_gaussian_noise_std4_fixed	.045
261	alexnet_early_checkpoint	.044
262	alexnet_l2_3_robust	.044
263	resnet_50_v1_spiking_l4	.042
264	mobilenet_v2_1.0_96	.042
265	mobilenet_v2_0.5_128	.038
266	alexnet_training_seed_05	.037
267	alexnet_training_seed_01	.037
268	mobilenet_v1_0.25_160	.035
269	imagenet_l2_10_0	.035
270	mobilenet_v1_0.5_160	.033
271	resnet50-vicregl0p75	.032
272	alexnet_linf_8_robust	.029
273	alexnet_robust_correct	.023
274	resnet-50_untrained	.021
275	alexnet_training_seed_06	.018
276	deit_small_imagenet_100_seed-0	.018
277	resnet50-meshes-lt-100-original-pretrained	.018
278	alexnet_training_seed_03	.016
279	resnet50-vicreg	.015
280	resnet50_FractalDB	.012
281	artResNet18_1	.011
282	alexnet_training_seed_07	.011
283	resnet50-vicregl0p9	.009
284	alexnet_training_seed_02	.009
285	alexnet_training_seed_04	.009
286	alexnet_training_seed_08	.009
287	alexnet_training_seed_10	.009
288	resnet50-meshes-lt-100-original-scratch	.009
289	mobilenet_v1_0.25_128	.008
290	deit_small_imagenet_10_seed-0	.008
291	deit_small_imagenet_1_seed-0	.008
292	convnext_small_imagenet_10_seed-0	.007
293	resnet50_imagenet_10_seed-0	.007
294	mobilevit_small	.007
295	resnet50_imagenet_1_seed-0	.006
296	resnet50-barlow	.004
297	resnet50-simclr-vissl	.003
298	barlow-twins-resnet50	.002
299	resnet_50_v1_spiking_l3	.001
300	alexnet_training_seed_09	.001
301	resnet-18-LC_w_sh_100_iter_conv_init_m
302	ViT-B/32
303	BiT-M-R50x3
304	BiT-M-R50x1
305	BiT-M-R152x2
306	resnet-18-LC_w_sh_1_iter_conv_init_m
307	resnet-18-LC_w_sh_10_iter_m
308	resnet-18-LC_w_sh_100_iter_m
309	resnet-18-LC_w_sh_10_iter_conv_init_m
310	FrankRobWobv0
311	resnet-18-LC_w_sh_1_iter_m
312	0.5x_resnet-18_LC
313	CORnetZ_CIFAR10
314	resnet18-local_aggregation
315	resnet18-deepcluster
316	r3m_resnet18
317	CORnet-Z
318	vggface
319	my-model
320	resnet18-instance_recognition
321	0.5x_resnet-18_LC_w_sh_10_iter
322	r3m_resnet50
323	r3m_resnet34
324	resnet-18-LC_m
325	r3m_resnet50_nocrop
326	resnet18-supervised
327	resnet-18-LC_1st_conv_m
328	resnet-18-LC_conv_init_m
329	resnet18-contrastive_multiview
330	0.5x_resnet-18
331	resnet18-colorization
332	RN50
333	0.5x_resnet-18_LC_w_sh_100_iter
334	RN50
335	resnet18-relative_position
336	resnet18-autoencoder
337	resnet18-simclr
338	ViT_L_32
339	ViT_B_32
340	resnet18-depth_prediction
341	ViT_B_16
342	r3m_resnet34_nocrop
343	resnet18-contrastive_predictive
344	resnet-18-LC_untrained
345	resnet-18_untrained
346	resnet-50x2_untrained
347	dcgan
348	unet_entire
349	v1-pyr-nodown
350	CORnetZ_CIFAR10_bs32_20_04
351	omnivore_swinT
352	pixels
353	dorinet_cornet_z
354	my-model
355	resnet50
356	openclip
357	resnet50-cifar
358	resnet50_pretrained_with_retinal_waves
359	resnet50_ImageNet
360	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
361	ReAlnet07_ALL
362	yudixie_resnet50_imagenet1kpret_0_240312
363	yudixie_resnet50_imagenet1kpret_0_240908
364	yudixie_resnet50_random_0_240908
365	yudixie_resnet50_cat_obj_class_all_latents_0_240908
366	yudixie_resnet50_category_class_0_240908
367	resnet18_imagenet21kP
368	ConvLSTM
369	PredRNN
370	yudixie_resnet50_distance_reg_0_240908
371	SimVP
372	TAU
373	MIM
374	hmax
375	inception_v3_pytorch
376	yolos_tiny
377	ReAlnet01_cornet
378	yudixie_resnet18_translation_reg_0_240719
379	yudixie_resnet18_distance_reg_0_240719
380	compact_resnet50_V4
381	yudixie_resnet18_rotation_reg_0_240719
382	briaai_rmbg_1_4
383	ReAlnet03_cornet
384	resnet18_ecoset_full
385	resnet34_ecoset_full
386	compact_resnet50_IT
387	resnet50_ecoset_full
388	compact_resnet50_robust_V4
389	resnet101_ecoset_full
390	yudixie_resnet50_distance_translation_0_240908
391	yudixie_resnet50_rotation_reg_0_240908
392	resnet152_ecoset_full
393	yudixie_resnet18_distance_translation_0_240719
394	yudixie_resnet18_translation_rotation_0_240719
395	yudixie_resnet18_distance_rotation_0_240719
396	yudixie_resnet18_category_class_0_240719
397	ReAlnet05_cornet
398	compact_vgg19_V4
399	yudixie_resnet18_cat_obj_class_all_latents_0_240719
400	resnet-18_test_m
401	yudixie_resnet18_distance_translation_rotation_0_240719
402	yudixie_resnet18_object_class_0_240719
403	yudixie_resnet18_imagenet1kpret_0_240719
404	SeLaVi-AVE
405	ReAlnet06_cornet
406	r3d_18
407	compact_vgg19_IT
408	yudixie_resnet50_translation_reg_0_240908
409	yudixie_resnet50_translation_rotation_0_240908
410	ReAlnet01_ALL
411	mc3_18
412	ReAlnet07_cornet
413	s3d
414	mvit_v1_b
415	ReAlnet08_cornet
416	ReAlnet03_ALL
417	VOneCORnet-S
418	compact_cornets_V4
419	omnivore_swinB
420	ReAlnet04_ALL
421	SeLaVi-Kinetics400
422	ReAlnet09_cornet
423	SeLaVi-VGG-Sound
424	compact_cornets_IT
425	ReAlnet05_ALL
426	omnivore_swinS
427	ReAlnet06_ALL
428	mvit_v2_s
429	SeLaVi-Kinetics-Sound
430	compact_resnet50_robust_IT
431	ReAlnet10_cornet
432	r2plus1d_18
433	ReAlnet08_ALL
434	yudixie_resnet18_random_0_240719
435	yudixie_resnet50_distance_rotation_0_240908
436	ReAlnet02_cornet
437	yudixie_resnet50_distance_translation_rotation_0_240908
438	ReAlnet09_ALL
439	ReAlnet04_cornet
440	resnet50_primary_visual_cortex
441	yudixie_resnet50_object_class_0_240908
442	ReAlnet10_ALL
443	ReAlnet02_ALL

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.44.

Note that scores are relative to this ceiling.

Data: Geirhos2021rotation

Metric: error_consistency