Leaderboard Explore Compare Profile Community Blog FAQ Competition 2024 Report Issue

Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021eidolonII-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	SWSL_resnext101_32x8d	.732
2	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.703
3	ViT_B_16_imagenet1k	.699
4	vision_transformer_vit_large_patch16_224	.696
5	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.692
6	vit_base_patch16_clip_224:openai_ft_in1k	.680
7	vit_relpos_base_patch16_clsgap_224:sw_in1k	.674
8	convnext_tiny_imagenet_full_seed-0	.665
9	resnext101_32x48d_wsl	.664
10	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.662
11	resnext101_32x16d_wsl	.662
12	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.659
13	AdvProp_efficientnet-b6	.656
14	resnext101_32x32d_wsl	.650
15	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.650
16	resnet_152_v2	.646
17	focalnet_tiny_lrf_in1k	.646
18	cvt_cvt-13-224-in1k_4_LucyV4	.643
19	pnasnet_large	.640
20	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.635
21	convnext_small_imagenet_100_seed-0	.632
22	resnet_50_v2	.630
23	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.626
24	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.625
25	resnext101_32x8d_wsl	.624
26	CLIP_ViT-B_32	.622
27	ViT_L_32_imagenet1k	.617
28	convnext_large:fb_in22k_ft_in1k	.613
29	vit_large_patch14_clip_224:laion2b_ft_in1k	.613
30	SWSL_resnet50	.613
31	resnet_101_v2	.610
32	vit_large_patch14_clip_224:openai_ft_in1k	.603
33	resnet-152_v2-tf	.602
34	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.597
35	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.590
36	convnext_femto_ols:d1_in1k	.588
37	cvt_cvt-13-384-in1k_4_LucyV4	.582
38	AdvProp_efficientnet-b7	.578
39	pnasnet_large-tf	.577
40	convnext_large_imagenet_full_seed-0	.569
41	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.564
42	AdvProp_efficientnet-b4	.559
43	AT_efficientnet-b4	.559
44	CLIP_resnet50	.558
45	CLIP_resnet50_float32	.558
46	convnext_base:clip_laiona_augreg_ft_in1k_384	.555
47	fixres_resnext101_32x48d_wsl	.549
48	convnext_tiny:in12k_ft_in1k	.547
49	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.546
50	effnetv2m_custom384	.540
51	efficientnet_b0	.538
52	cvt_cvt-21-384-in1k_4_LucyV4	.535
53	efficientnet-b7	.534
54	nasnet_large	.534
55	resnet152_imagenet_full	.527
56	resnet_152_v1	.524
57	resnet50-VITO-8deg-cc	.522
58	nasnet_large-tf	.520
59	cvt_cvt-21-224-in1k_4_LucyV4	.519
60	ViT_B_32_imagenet1k	.518
61	antialias-resnet152	.515
62	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.513
63	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.512
64	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.509
65	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.503
66	AT_efficientnet-b2	.501
67	AdvProp_efficientnet-b2	.501
68	deit_large_imagenet_full_seed-0	.496
69	effnetb1_272x240	.495
70	convnext_xlarge:fb_in22k_ft_in1k	.491
71	inception_v4	.482
72	vit_large_patch14_clip_336:laion2b_ft_in1k	.481
73	CLIP-RN50	.477
74	convnext_base_imagenet_full_seed-0	.475
75	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.473
76	resnet_101_v1	.470
77	antialiased-rnext101_32x8d	.469
78	convnext_small_imagenet_full_seed-0	.465
79	deit_base_imagenet_full_seed-0	.465
80	inception_v3_pytorch	.463
81	inception_v3	.458
82	resnet-152_v1-tf	.455
83	grcnn_109	.454
84	tv_efficientnet-b1	.454
85	resnet50-moclr8deg	.453
86	resnet-152_v2_pytorch	.453
87	resnet101	.451
88	resnet50_imagenet_full	.448
89	pnasnet_large_pytorch	.445
90	cv_18_dagger_408_pretrained	.443
91	resnet101_imagenet_full	.433
92	densenet-201-keras	.423
93	inception_v4-tf	.423
94	densenet-201	.421
95	densenet_201_pytorch	.421
96	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.416
97	resnet-50_v2-tf	.416
98	grcnn	.414
99	deit_small_imagenet_full_seed-0	.410
100	resnet50_random_linf8_perturb	.405
101	custom_model_cv_18_dagger_408	.401
102	xception-keras	.399
103	BiT-S-R50x3	.399
104	Res2Net50_26w_4s	.394
105	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.379
106	resnet34_imagenet_full	.379
107	inception_v3-tf	.375
108	BiT-S-R152x2	.371
109	resnet50_random_l2_perturb	.371
110	ReAlnet03	.366
111	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.365
112	resnet50_moco_v2	.364
113	resnet50	.363
114	cornet_s	.361
115	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.361
116	vonegrcnn_62e_nobn	.359
117	efficientnet_b2_imagenet_full	.359
118	resnet50_linf_4_robust	.358
119	vonegrcnn_47e	.354
120	vonegrcnn_52e_full	.351
121	AdvProp_efficientnet-b8	.350
122	ReAlnet01	.345
123	ReAlnet04	.342
124	resnet50_byol	.337
125	BiT-S-R101x3	.331
126	vonegrcnn_47e	.330
127	resnet50_tutorial	.329
128	resnet_50_v1	.329
129	resnet-50-pytorch	.329
130	resnet50-sup	.329
131	resnet18_imagenet_full	.329
132	vonegrcnn_52e_full	.325
133	densenet-121-keras	.320
134	resnet-50_v1-tf	.317
135	vgg_19	.317
136	antialiased-r50	.315
137	alexnet_random_linf8_perturb	.314
138	resnet50-SIN_IN_IN	.313
139	resnet_SIN_IN_FT_IN	.313
140	ReAlnet06	.313
141	densenet-169-keras	.311
142	vonegrcnn_62e_nobn	.311
143	alexnet_reduced_aliasing_early_checkpoint	.308
144	CORnet-S	.304
145	densenet-121	.302
146	ReAlnet05	.299
147	ReAlnet07	.298
148	xception	.298
149	resnet_50_v1_spiking	.297
150	voneresnet-50-robust	.296
151	mobilenet_v2_1.4_224-tf	.296
152	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.294
153	inception_v2-tf	.294
154	voneresnet-50	.289
155	inception_v1-tf	.289
156	mobilenet_v2_1-4_224_pytorch	.287
157	mobilenet_v2_1_4_224	.287
158	voneresnet-50-non_stochastic	.286
159	resnet-101_v1-tf	.285
160	ReAlnet10	.279
161	ReAlnet08	.278
162	ReAlnet02	.274
163	densenet-169	.270
164	vgg_16	.270
165	mobilenet_v2_1_0_224	.267
166	resnet50_l2_3_robust	.266
167	mobilenet_v2_1_0_160	.266
168	ReAlnet09	.265
169	mobilenet_v2_1_3_224	.260
170	alexnet	.260
171	resnet50-SIN_IN	.258
172	BiT-S-R152x4	.257
173	BiT-S-R50x1	.256
174	alexnet_l2_3_robust	.255
175	resnet50-vitoimagevidnet8	.254
176	alexnet-baseline	.252
177	alexnet	.252
178	alexnet	.252
179	alexnet	.252
180	alexnet	.252
181	alexnet	.252
182	alexnet	.252
183	alexnet	.252
184	alexnet	.252
185	alexnet	.252
186	alexnet_ks_torevert	.252
187	nasnet_mobile	.250
188	blt_vs	.247
189	alexnet_linf_8_robust	.246
190	alexnet_robust_correct	.236
191	vonealexnet_gaussian_noise_std4_fixed	.235
192	inception_v1	.235
193	mobilenet_v2_0_5_224	.226
194	resnet-50-robust	.226
195	imagenet_l2_3_0	.226
196	resnet50_robust_l2_eps3	.226
197	mobilenet_v2_1.0_160-tf	.225
198	deit_small_imagenet_100_seed-0	.224
199	efficientnet_b0_imagenet_full	.224
200	konkle_alexnetgn_ipcl_ref01_primary_model	.223
201	convnext_tiny_sup	.223
202	resnet-18	.216
203	resnet-34	.216
204	alexnet_random_l2_3_perturb	.215
205	resnet50_robust_l2_eps1	.206
206	mobilenet_v2_1_0_192	.206
207	vgg-16-keras	.204
208	mobilenet_v2_1.3_224-tf	.198
209	mobilenet_v1_1.0_224	.196
210	resnet50_linf_8_robust	.195
211	BiT-S-R101x1	.191
212	regnet_y_400mf	.190
213	efficientnet_b1_imagenet_full	.189
214	mobilenet_v2_1.0_192-tf	.189
215	resnet50_imagenet_10_seed-0	.182
216	deit_small_imagenet_10_seed-0	.182
217	mobilenet_v1_0.75_224	.180
218	mobilenet_v2_1.0_224-tf	.180
219	resnet50-barlow	.179
220	resnet50-vicreg	.178
221	texture_shape_resnet50_trained_on_SIN	.178
222	deit_small_imagenet_1_seed-0	.174
223	resnet50_eMMCR_Vanilla	.173
224	nasnet_mobile-tf	.169
225	mobilenet_v2_0_5_192	.166
226	convnext_small_imagenet_10_seed-0	.161
227	mobilenet_v2_0.5_192-tf	.161
228	resnet50_simclr	.158
229	shufflenet_v2_x1_0	.158
230	mobilenet_v2_0_75_160	.155
231	mobilenet_v1_1.0_192	.154
232	alexnet_early_checkpoint	.151
233	imagenet_l2_10_0	.149
234	mobilenet_v1_0.75_128	.145
235	mobilenet_v2_0.5_224-tf	.143
236	alexnet_training_seed_05	.143
237	alexnet_training_seed_01	.143
238	0.5x_resnet-18_LC_w_sh_1_iter	.142
239	mobilenet_v2_0.35_160	.137
240	vgg-19-keras	.136
241	resnet50-SIN	.134
242	mobilenet_v2_0.5_96	.133
243	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.128
244	texture_shape_alexnet_trained_on_SIN	.124
245	mobilenet_v2_0.75_192-tf	.124
246	AdvProp_efficientnet-b0	.118
247	resnet50_imagenet_100_seed-0	.118
248	mobilenet_v1_0.75_192	.116
249	AlexNet_SIN	.115
250	mobilenet_v2_0.75_224-tf	.114
251	mobilenet_v2_0.5_160	.114
252	mobilenet_v1_0.25_160	.112
253	mobilenet_v2_0_75_192	.110
254	mobilenet_v2_0.35_128	.109
255	mobilenet_v2_0_75_224	.107
256	mobilenet_v2_0.75_128	.105
257	mobilenet_v2_1_0_128	.105
258	mobilenet_v2_0.35_96	.104
259	mobilenet_v1_0.75_160	.100
260	alexnet_training_seed_06	.098
261	mobilenet_v1_1.0_160	.098
262	mobilenet_v1_0.5_192	.096
263	mobilenet_v2_1.0_96	.089
264	mobilenet_v2_1.0_128-tf	.088
265	mobilenet_v2_0.75_160-tf	.086
266	mobilenet_v1_0.5_224	.086
267	mobilenet_v1_0.25_192	.086
268	mobilenet_v2_0.35_192	.078
269	squeezenet1_1	.075
270	mobilenet_v1_1.0_128	.071
271	my-model	.065
272	mobilenet_v2_0.75_96	.065
273	resnet_50_v1_spiking_l4a	.061
274	mobilenet_v2_0.35_224	.061
275	squeezenet1_0	.058
276	resnet50-simclr-vissl	.058
277	resnet50-vicregl0p9	.057
278	mobilevit_small	.052
279	alexnet_training_seed_02	.049
280	alexnet_training_seed_04	.049
281	alexnet_training_seed_08	.049
282	alexnet_training_seed_10	.049
283	resnet50-vicregl0p75	.048
284	mobilenet_v1_0.25_128	.046
285	resnet50-meshes-lt-100-original-scratch	.043
286	mobilenet_v2_0.5_128	.042
287	resnet-50_untrained	.041
288	mobilenet_v1_0.5_160	.040
289	mobilenet_v1_0.25_224	.039
290	convnext_small_imagenet_1_seed-0	.032
291	alexnet_training_seed_03	.030
292	resnet50-meshes-lt-100-original-pretrained	.030
293	resnet_50_v1_spiking_l3	.026
294	resnet50_FractalDB	.024
295	alexnet_training_seed_07	.024
296	resnet_50_v1_spiking_l4	.020
297	alexnet_training_seed_09	.013
298	resnet50_imagenet_1_seed-0	.010
299	mobilenet_v1_0.5_128	.007
300	artResNet18_1	.006
301	barlow-twins-resnet50	.002
302	resnet-18-LC_w_sh_100_iter_conv_init_m
303	ViT-B/32
304	BiT-M-R50x3
305	BiT-M-R50x1
306	BiT-M-R152x2
307	resnet-18-LC_w_sh_1_iter_conv_init_m
308	resnet-18-LC_w_sh_10_iter_m
309	resnet-18-LC_w_sh_100_iter_m
310	resnet-18-LC_w_sh_1_iter_m
311	resnet-18-LC_w_sh_10_iter_conv_init_m
312	FrankRobWobv0
313	resnet18-local_aggregation
314	0.5x_resnet-18_LC
315	CORnetZ_CIFAR10
316	resnet18-deepcluster
317	r3m_resnet18
318	CORnet-Z
319	resnet18-instance_recognition
320	my-model
321	vggface
322	0.5x_resnet-18_LC_w_sh_10_iter
323	resnet18-colorization
324	r3m_resnet50
325	r3m_resnet34
326	resnet-18-LC_m
327	r3m_resnet50_nocrop
328	resnet18-supervised
329	resnet-18-LC_1st_conv_m
330	resnet-18-LC_conv_init_m
331	resnet18-contrastive_multiview
332	0.5x_resnet-18
333	RN50
334	0.5x_resnet-18_LC_w_sh_100_iter
335	RN50
336	resnet18-relative_position
337	resnet18-autoencoder
338	resnet18-simclr
339	ViT_L_32
340	ViT_B_32
341	resnet18-depth_prediction
342	ViT_B_16
343	r3m_resnet34_nocrop
344	resnet18-contrastive_predictive
345	omnivore_swinT
346	resnet-18-LC_untrained
347	resnet-18_untrained
348	resnet-50x2_untrained
349	dcgan
350	v1-pyr-nodown
351	unet_entire
352	yudixie_resnet50_imagenet1kpret_0_240908
353	CORnetZ_CIFAR10_bs32_20_04
354	pixels
355	dorinet_cornet_z
356	resnet50
357	openclip
358	resnet50-cifar
359	resnet50_pretrained_with_retinal_waves
360	resnet50_ImageNet
361	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
362	yudixie_resnet50_random_0_240908
363	yudixie_resnet50_cat_obj_class_all_latents_0_240908
364	yudixie_resnet50_imagenet1kpret_0_240312
365	yudixie_resnet50_object_class_0_240908
366	yudixie_resnet50_category_class_0_240908
367	resnet18_imagenet21kP
368	yudixie_resnet50_distance_reg_0_240908
369	ConvLSTM
370	PredRNN
371	SimVP
372	TAU
373	MIM
374	hmax
375	ReAlnet01_cornet
376	briaai_rmbg_1_4
377	yolos_tiny
378	yudixie_resnet18_translation_reg_0_240719
379	yudixie_resnet18_distance_reg_0_240719
380	ReAlnet03_cornet
381	yudixie_resnet18_rotation_reg_0_240719
382	yudixie_resnet18_imagenet1kpret_0_240719
383	ReAlnet02_cornet
384	resnet18_ecoset_full
385	VOneCORnet-S
386	resnet50_ecoset_full
387	yudixie_resnet50_distance_rotation_0_240908
388	yudixie_resnet50_rotation_reg_0_240908
389	resnet152_ecoset_full
390	yudixie_resnet50_distance_translation_0_240908
391	yudixie_resnet18_distance_translation_0_240719
392	yudixie_resnet18_translation_rotation_0_240719
393	yudixie_resnet18_distance_rotation_0_240719
394	yudixie_resnet18_category_class_0_240719
395	ReAlnet05_cornet
396	resnet-18_test_m
397	yudixie_resnet18_cat_obj_class_all_latents_0_240719
398	yudixie_resnet18_distance_translation_rotation_0_240719
399	yudixie_resnet18_object_class_0_240719
400	SeLaVi-AVE
401	ReAlnet06_cornet
402	r3d_18
403	yudixie_resnet50_translation_reg_0_240908
404	yudixie_resnet50_translation_rotation_0_240908
405	ReAlnet01_ALL
406	ReAlnet07_cornet
407	mc3_18
408	s3d
409	mvit_v1_b
410	ReAlnet08_cornet
411	ReAlnet02_ALL
412	ReAlnet03_ALL
413	ReAlnet04_ALL
414	omnivore_swinB
415	SeLaVi-Kinetics400
416	SeLaVi-VGG-Sound
417	ReAlnet05_ALL
418	SeLaVi-Kinetics-Sound
419	ReAlnet06_ALL
420	omnivore_swinS
421	r2plus1d_18
422	ReAlnet10_cornet
423	mvit_v2_s
424	ReAlnet07_ALL
425	resnet101_ecoset_full
426	resnet34_ecoset_full
427	yudixie_resnet50_distance_translation_rotation_0_240908
428	ReAlnet08_ALL
429	yudixie_resnet18_random_0_240719
430	ReAlnet04_cornet
431	ReAlnet09_ALL
432	ReAlnet09_cornet
433	resnet50_primary_visual_cortex
434	ReAlnet10_ALL

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.45.

Note that scores are relative to this ceiling.

Data: Geirhos2021eidolonII

Metric: error_consistency