Leaderboard Explore Compare Profile Community Blog FAQ Competition 2024 Report Issue

Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021rotation-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.646
2	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.614
3	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.606
4	resnet-152_v2_pytorch	.589
5	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.581
6	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.571
7	fixres_resnext101_32x48d_wsl	.568
8	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.565
9	resnet_152_v2	.561
10	vit_large_patch14_clip_336:laion2b_ft_in1k	.559
11	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.546
12	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.539
13	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.528
14	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.527
15	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.516
16	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.506
17	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.500
18	effnetv2m_custom384	.489
19	vit_large_patch14_clip_224:openai_ft_in1k	.488
20	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.478
21	convnext_base:clip_laiona_augreg_ft_in1k_384	.476
22	convnext_large:fb_in22k_ft_in1k	.467
23	vit_large_patch14_clip_224:laion2b_ft_in1k	.457
24	pnasnet_large	.431
25	vit_base_patch16_clip_224:openai_ft_in1k	.429
26	custom_model_cv_18_dagger_408	.425
27	convnext_xlarge:fb_in22k_ft_in1k	.417
28	resnet_101_v2	.416
29	resnet_50_v2	.413
30	resnext101_32x48d_wsl	.405
31	CLIP_resnet50	.404
32	CLIP_resnet50_float32	.404
33	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.404
34	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.400
35	resnext101_32x16d_wsl	.391
36	resnext101_32x32d_wsl	.385
37	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.380
38	SWSL_resnet50	.370
39	SWSL_resnext101_32x8d	.368
40	CLIP-RN50	.367
41	pnasnet_large-tf	.357
42	effnetb1_272x240	.356
43	resnext101_32x8d_wsl	.351
44	nasnet_large	.344
45	cvt_cvt-13-384-in1k_4_LucyV4	.343
46	cvt_cvt-21-384-in1k_4_LucyV4	.338
47	convnext_tiny:in12k_ft_in1k	.334
48	nasnet_large-tf	.325
49	vision_transformer_vit_large_patch16_224	.308
50	antialias-resnet152	.282
51	inception_v3	.273
52	CLIP_ViT-B_32	.273
53	focalnet_tiny_lrf_in1k	.273
54	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.268
55	inception_v4	.266
56	inception_v3-tf	.264
57	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.261
58	resnet152_imagenet_full	.259
59	convnext_femto_ols:d1_in1k	.255
60	resnet101	.255
61	resnet50-VITO-8deg-cc	.254
62	resnet101_imagenet_full	.250
63	vit_relpos_base_patch16_clsgap_224:sw_in1k	.249
64	resnet-152_v2-tf	.239
65	convnext_large_imagenet_full_seed-0	.236
66	resnet_152_v1	.236
67	cv_18_dagger_408_pretrained	.226
68	xception-keras	.225
69	antialiased-rnext101_32x8d	.225
70	convnext_tiny_imagenet_full_seed-0	.224
71	resnet50_random_l2_perturb	.223
72	tv_efficientnet-b1	.221
73	cvt_cvt-13-224-in1k_4_LucyV4	.218
74	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.213
75	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.212
76	convnext_base_imagenet_full_seed-0	.206
77	resnet50	.206
78	inception_v4-tf	.204
79	resnet50_imagenet_full	.204
80	deit_base_imagenet_full_seed-0	.201
81	convnext_small_imagenet_full_seed-0	.197
82	deit_large_imagenet_full_seed-0	.196
83	resnet_101_v1	.195
84	antialiased-r50	.195
85	grcnn_109	.194
86	resnet50-moclr8deg	.190
87	deit_small_imagenet_full_seed-0	.190
88	ViT_B_16_imagenet1k	.182
89	cvt_cvt-21-224-in1k_4_LucyV4	.181
90	densenet-201-keras	.181
91	xception	.179
92	Res2Net50_26w_4s	.177
93	pnasnet_large_pytorch	.175
94	densenet-201	.172
95	densenet_201_pytorch	.172
96	resnet_50_v1_spiking	.172
97	efficientnet_b2_imagenet_full	.167
98	vgg_19	.166
99	efficientnet_b0	.165
100	densenet-121-keras	.164
101	convnext_tiny_sup	.161
102	ViT_L_32_imagenet1k	.161
103	grcnn	.161
104	resnet34_imagenet_full	.160
105	resnet50_ewconplus_wzc	.160
106	resnet-50_v2-tf	.156
107	densenet-121	.155
108	resnet50_tutorial	.154
109	resnet_50_v1	.154
110	resnet-50-pytorch	.154
111	resnet50-sup	.154
112	mobilenet_v2_1.4_224-tf	.153
113	resnet50_random_linf8_perturb	.152
114	ViT_B_32_imagenet1k	.152
115	AdvProp_efficientnet-b6	.150
116	resnet50-SIN_IN_IN	.150
117	resnet_SIN_IN_FT_IN	.150
118	BiT-S-R50x3	.149
119	inception_v2-tf	.149
120	resnet-101_v1-tf	.149
121	resnet50_byol	.144
122	efficientnet-b7	.143
123	nasnet_mobile	.143
124	mobilenet_v2_1_0_224	.140
125	AT_efficientnet-b2	.137
126	AdvProp_efficientnet-b2	.137
127	alexnet_fov4	.136
128	resnet50-vitoimagevidnet8	.135
129	cornet_s	.133
130	BiT-S-R152x2	.132
131	densenet-169-keras	.131
132	voneresnet-50-non_stochastic	.130
133	BiT-S-R101x3	.130
134	mobilenet_v1_0.75_224	.130
135	mobilenet_v2_1.0_224-tf	.130
136	resnet18_imagenet_full	.130
137	efficientnet_b0_imagenet_full	.129
138	mobilenet_v2_1_3_224	.124
139	ReAlnet04	.124
140	AdvProp_efficientnet-b7	.123
141	inception_v1	.123
142	nasnet_mobile-tf	.121
143	ReAlnet03	.120
144	resnet50-SIN_IN	.119
145	resnet-50_v1-tf	.118
146	convnext_small_imagenet_100_seed-0	.117
147	ReAlnet10	.116
148	resnet_50_v1_spiking_l4a	.116
149	vgg_16	.116
150	AdvProp_efficientnet-b4	.115
151	AT_efficientnet-b4	.115
152	ReAlnet06	.115
153	ReAlnet01	.114
154	ReAlnet02	.114
155	ReAlnet09	.114
156	resnet-152_v1-tf	.114
157	densenet-169	.112
158	mobilenet_v2_1.3_224-tf	.109
159	resnet50_moco_v2	.105
160	BiT-S-R152x4	.105
161	efficientnet_b1_imagenet_full	.101
162	AlexNet_SIN_fov4	.101
163	mobilenet_v1_1.0_192	.100
164	inception_v1-tf	.098
165	ReAlnet05	.098
166	mobilenet_v2_1.0_160-tf	.096
167	vonegrcnn_47e	.096
168	ReAlnet07	.095
169	mobilenet_v2_1_0_192	.094
170	CORnet-S	.094
171	mobilenet_v1_1.0_224	.092
172	ReAlnet08	.092
173	mobilenet_v2_1_0_160	.091
174	mobilenet_v2_0_5_224	.091
175	resnet-34	.090
176	mobilenet_v2_0_75_192	.089
177	BiT-S-R50x1	.089
178	mobilenet_v2_0_75_160	.088
179	mobilenet_v2_1-4_224_pytorch	.087
180	mobilenet_v2_1_4_224	.087
181	resnet50_linf_4_robust	.085
182	squeezenet1_1	.085
183	BiT-S-R101x1	.085
184	vonegrcnn_62e_nobn	.085
185	regnet_y_400mf	.085
186	vgg-16-keras	.084
187	vonegrcnn_47e	.081
188	resnet50_simclr	.081
189	mobilenet_v2_0_75_224	.080
190	vonegrcnn_52e_full	.080
191	resnet-18	.079
192	vonegrcnn_52e_full	.079
193	mobilenet_v2_0_5_192	.079
194	AdvProp_efficientnet-b0	.078
195	vonegrcnn_62e_nobn	.075
196	shufflenet_v2_x1_0	.074
197	resnet50_l2_3_robust	.073
198	mobilenet_v1_0.5_192	.073
199	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.073
200	0.5x_resnet-18_LC_w_sh_1_iter	.070
201	mobilenet_v2_0.75_160-tf	.070
202	alexnet	.069
203	mobilenet_v2_1.0_192-tf	.069
204	texture_shape_alexnet_trained_on_SIN	.068
205	mobilenet_v2_0.75_224-tf	.067
206	mobilenet_v2_0.75_192-tf	.064
207	texture_shape_resnet50_trained_on_SIN	.064
208	mobilenet_v2_0.35_192	.064
209	mobilenet_v2_1_0_128	.063
210	mobilenet_v2_0.75_96	.063
211	resnet50_eMMCR_Vanilla	.063
212	mobilenet_v1_0.25_224	.062
213	squeezenet1_0	.062
214	resnet50_robust_l2_eps1	.062
215	resnet50_linf_8_robust	.061
216	mobilenet_v2_0.5_192-tf	.061
217	mobilenet_v1_1.0_128	.061
218	AdvProp_efficientnet-b8	.061
219	mobilenet_v1_0.25_192	.060
220	mobilenet_v1_0.75_160	.060
221	mobilenet_v2_0.35_224	.059
222	blt_vs	.059
223	konkle_alexnetgn_ipcl_ref01_primary_model	.059
224	mobilenet_v1_0.5_224	.058
225	resnet50_imagenet_100_seed-0	.058
226	alexnet_random_linf8_perturb	.058
227	resnet-50-robust	.058
228	imagenet_l2_3_0	.058
229	resnet50_robust_l2_eps3	.058
230	voneresnet-50	.057
231	vgg-19-keras	.057
232	alexnet_reduced_aliasing_early_checkpoint	.056
233	mobilenet_v2_0.5_224-tf	.055
234	mobilenet_v2_0.75_128	.055
235	mobilenet_v2_0.35_96	.055
236	resnet50-SIN	.055
237	convnext_small_imagenet_1_seed-0	.054
238	mobilenet_v2_1.0_128-tf	.054
239	mobilenet_v1_0.75_128	.053
240	AlexNet_SIN	.052
241	mobilenet_v2_0.35_160	.052
242	mobilenet_v1_0.75_192	.052
243	voneresnet-50-robust	.052
244	mobilenet_v1_1.0_160	.050
245	alexnet_random_l2_3_perturb	.050
246	mobilenet_v2_0.5_96	.050
247	mobilenet_v2_0.5_160	.049
248	alexnet_ks_torevert	.049
249	alexnet-baseline	.049
250	alexnet	.049
251	alexnet	.049
252	alexnet	.049
253	alexnet	.049
254	alexnet	.049
255	alexnet	.049
256	alexnet	.049
257	alexnet	.049
258	alexnet	.049
259	alexnet	.049
260	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.049
261	mobilenet_v2_0.35_128	.048
262	mobilenet_v1_0.5_128	.046
263	alexnet_fov12	.045
264	vonealexnet_gaussian_noise_std4_fixed	.045
265	alexnet_early_checkpoint	.044
266	alexnet_l2_3_robust	.044
267	resnet_50_v1_spiking_l4	.042
268	AlexNet_SIN_fov12	.042
269	mobilenet_v2_1.0_96	.042
270	alexnet_fov16	.041
271	mobilenet_v2_0.5_128	.038
272	alexnet_training_seed_05	.037
273	alexnet_training_seed_01	.037
274	alexnet_training_seed_01_fov4	.037
275	alexnet_training_seed_01_fov12	.037
276	alexnet_training_seed_01_fov16	.037
277	mobilenet_v1_0.25_160	.035
278	imagenet_l2_10_0	.035
279	mobilenet_v1_0.5_160	.033
280	resnet50-vicregl0p75	.032
281	AlexNet_SIN_fov16	.031
282	alexnet_linf_8_robust	.029
283	alexnet_robust_correct	.023
284	resnet-50_untrained	.021
285	alexnet_training_seed_06	.018
286	deit_small_imagenet_100_seed-0	.018
287	resnet50-meshes-lt-100-original-pretrained	.018
288	alexnet_training_seed_03	.016
289	resnet50-vicreg	.015
290	resnet50_FractalDB	.012
291	artResNet18_1	.011
292	alexnet_training_seed_07	.011
293	resnet50-vicregl0p9	.009
294	alexnet_training_seed_02	.009
295	alexnet_training_seed_04	.009
296	alexnet_training_seed_08	.009
297	alexnet_training_seed_10	.009
298	resnet50-meshes-lt-100-original-scratch	.009
299	mobilenet_v1_0.25_128	.008
300	deit_small_imagenet_10_seed-0	.008
301	deit_small_imagenet_1_seed-0	.008
302	convnext_small_imagenet_10_seed-0	.007
303	resnet50_imagenet_10_seed-0	.007
304	mobilevit_small	.007
305	resnet50_imagenet_1_seed-0	.006
306	resnet50-barlow	.004
307	resnet50-simclr-vissl	.003
308	barlow-twins-resnet50	.002
309	resnet_50_v1_spiking_l3	.001
310	alexnet_training_seed_09	.001
311	resnet-18-LC_w_sh_100_iter_conv_init_m
312	ViT-B/32
313	BiT-M-R50x3
314	BiT-M-R50x1
315	BiT-M-R152x2
316	resnet-18-LC_w_sh_1_iter_conv_init_m
317	resnet-18-LC_w_sh_10_iter_m
318	resnet-18-LC_w_sh_100_iter_m
319	resnet-18-LC_w_sh_10_iter_conv_init_m
320	FrankRobWobv0
321	resnet-18-LC_w_sh_1_iter_m
322	0.5x_resnet-18_LC
323	CORnetZ_CIFAR10
324	resnet18-local_aggregation
325	resnet18-deepcluster
326	r3m_resnet18
327	CORnet-Z
328	vggface
329	my-model
330	resnet18-instance_recognition
331	0.5x_resnet-18_LC_w_sh_10_iter
332	r3m_resnet50
333	r3m_resnet34
334	resnet-18-LC_m
335	r3m_resnet50_nocrop
336	resnet18-supervised
337	resnet-18-LC_1st_conv_m
338	resnet-18-LC_conv_init_m
339	resnet18-contrastive_multiview
340	0.5x_resnet-18
341	resnet18-colorization
342	RN50
343	0.5x_resnet-18_LC_w_sh_100_iter
344	RN50
345	resnet18-relative_position
346	resnet18-autoencoder
347	resnet18-simclr
348	ViT_L_32
349	ViT_B_32
350	resnet18-depth_prediction
351	ViT_B_16
352	r3m_resnet34_nocrop
353	resnet18-contrastive_predictive
354	resnet-18-LC_untrained
355	resnet-18_untrained
356	resnet-50x2_untrained
357	dcgan
358	unet_entire
359	v1-pyr-nodown
360	CORnetZ_CIFAR10_bs32_20_04
361	omnivore_swinT
362	pixels
363	dorinet_cornet_z
364	my-model
365	resnet50
366	openclip
367	resnet50-cifar
368	resnet50_pretrained_with_retinal_waves
369	resnet50_ImageNet
370	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
371	ReAlnet07_ALL
372	yudixie_resnet50_imagenet1kpret_0_240312
373	yudixie_resnet50_imagenet1kpret_0_240908
374	yudixie_resnet50_random_0_240908
375	yudixie_resnet50_cat_obj_class_all_latents_0_240908
376	yudixie_resnet50_category_class_0_240908
377	resnet18_imagenet21kP
378	ConvLSTM
379	PredRNN
380	yudixie_resnet50_distance_reg_0_240908
381	SimVP
382	TAU
383	MIM
384	hmax
385	inception_v3_pytorch
386	yolos_tiny
387	ReAlnet01_cornet
388	yudixie_resnet18_translation_reg_0_240719
389	yudixie_resnet18_distance_reg_0_240719
390	compact_resnet50_V4
391	yudixie_resnet18_rotation_reg_0_240719
392	briaai_rmbg_1_4
393	ReAlnet03_cornet
394	resnet18_ecoset_full
395	ReAlnet01_cornet_fov4
396	resnet34_ecoset_full
397	compact_resnet50_IT
398	resnet50_ecoset_full
399	compact_resnet50_robust_V4
400	resnet101_ecoset_full
401	yudixie_resnet50_distance_translation_0_240908
402	yudixie_resnet50_rotation_reg_0_240908
403	resnet152_ecoset_full
404	ReAlnet01_cornet_fov12
405	yudixie_resnet18_distance_translation_0_240719
406	yudixie_resnet18_translation_rotation_0_240719
407	yudixie_resnet18_distance_rotation_0_240719
408	yudixie_resnet18_category_class_0_240719
409	ReAlnet05_cornet
410	compact_vgg19_V4
411	yudixie_resnet18_cat_obj_class_all_latents_0_240719
412	resnet-18_test_m
413	yudixie_resnet18_distance_translation_rotation_0_240719
414	yudixie_resnet18_object_class_0_240719
415	yudixie_resnet18_imagenet1kpret_0_240719
416	SeLaVi-AVE
417	ReAlnet06_cornet
418	r3d_18
419	ReAlnet01_cornet_fov16
420	compact_vgg19_IT
421	yudixie_resnet50_translation_reg_0_240908
422	yudixie_resnet50_translation_rotation_0_240908
423	ReAlnet01_ALL
424	mc3_18
425	ReAlnet07_cornet
426	s3d
427	mvit_v1_b
428	ReAlnet08_cornet
429	ReAlnet03_ALL
430	VOneCORnet-S
431	compact_cornets_V4
432	omnivore_swinB
433	ReAlnet04_ALL
434	SeLaVi-Kinetics400
435	ReAlnet09_cornet
436	SeLaVi-VGG-Sound
437	compact_cornets_IT
438	ReAlnet05_ALL
439	omnivore_swinS
440	ReAlnet06_ALL
441	mvit_v2_s
442	SeLaVi-Kinetics-Sound
443	compact_resnet50_robust_IT
444	ReAlnet10_cornet
445	r2plus1d_18
446	ReAlnet08_ALL
447	yudixie_resnet18_random_0_240719
448	yudixie_resnet50_distance_rotation_0_240908
449	ReAlnet02_cornet
450	yudixie_resnet50_distance_translation_rotation_0_240908
451	ReAlnet09_ALL
452	ReAlnet04_cornet
453	resnet50_primary_visual_cortex
454	yudixie_resnet50_object_class_0_240908
455	ReAlnet10_ALL
456	ReAlnet02_ALL

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.44.

Note that scores are relative to this ceiling.

Data: Geirhos2021rotation

Metric: error_consistency