Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021eidolonI-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	vision_transformer_vit_large_patch16_224	.789
2	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.783
3	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.777
4	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.776
5	resnext101_32x16d_wsl	.731
6	vit_base_patch16_clip_224:openai_ft_in1k	.724
7	SWSL_resnext101_32x8d	.715
8	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.698
9	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.689
10	ViT_B_16_imagenet1k	.677
11	resnext101_32x48d_wsl	.667
12	resnet_101_v2	.663
13	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.661
14	resnext101_32x8d_wsl	.655
15	CLIP_ViT-B_32	.651
16	SWSL_resnet50	.647
17	AdvProp_efficientnet-b7	.646
18	resnet_152_v2	.638
19	ViT_L_32_imagenet1k	.637
20	resnext101_32x32d_wsl	.637
21	vit_large_patch14_clip_224:laion2b_ft_in1k	.636
22	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.629
23	resnet101	.629
24	effnetv2m_custom384	.621
25	AdvProp_efficientnet-b4	.619
26	AT_efficientnet-b4	.619
27	vit_large_patch14_clip_224:openai_ft_in1k	.619
28	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.615
29	effnetb1_272x240	.614
30	vit_relpos_base_patch16_clsgap_224:sw_in1k	.610
31	resnet152_imagenet_full	.607
32	focalnet_tiny_lrf_in1k	.603
33	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.593
34	pnasnet_large	.593
35	pnasnet_large-tf	.584
36	efficientnet_b0	.583
37	fixres_resnext101_32x48d_wsl	.582
38	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.577
39	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.575
40	vit_large_patch14_clip_336:laion2b_ft_in1k	.574
41	convnext_xlarge:fb_in22k_ft_in1k	.574
42	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.567
43	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.556
44	AdvProp_efficientnet-b6	.553
45	cvt_cvt-13-384-in1k_4_LucyV4	.551
46	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.549
47	resnet_101_v1	.546
48	AdvProp_efficientnet-b2	.541
49	AT_efficientnet-b2	.541
50	cvt_cvt-13-224-in1k_4_LucyV4	.537
51	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.529
52	vonegrcnn_62e_nobn	.527
53	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.527
54	antialias-resnet152	.524
55	ViT_B_32_imagenet1k	.522
56	resnet-152_v2-tf	.520
57	inception_v4	.519
58	CLIP_resnet50	.513
59	CLIP_resnet50_float32	.513
60	custom_model_cv_18_dagger_408	.512
61	nasnet_large	.505
62	nasnet_large-tf	.499
63	convnext_tiny_imagenet_full_seed-0	.498
64	vonegrcnn_47e	.497
65	convnext_large:fb_in22k_ft_in1k	.492
66	resnet_50_v2	.491
67	vonegrcnn_52e_full	.487
68	convnext_femto_ols:d1_in1k	.482
69	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.481
70	vonegrcnn_62e_nobn	.476
71	convnext_base:clip_laiona_augreg_ft_in1k_384	.472
72	convnext_tiny:in12k_ft_in1k	.471
73	resnet_152_v1	.470
74	cvt_cvt-21-384-in1k_4_LucyV4	.464
75	resnet50_linf_4_robust	.459
76	resnet-152_v2_pytorch	.459
77	resnet-152_v1-tf	.459
78	vonegrcnn_47e	.456
79	efficientnet-b7	.453
80	densenet-201-keras	.448
81	densenet_201_pytorch	.446
82	inception_v3_pytorch	.445
83	inception_v4-tf	.443
84	resnet50_imagenet_full	.443
85	densenet-201	.443
86	resnet34_imagenet_full	.442
87	antialiased-rnext101_32x8d	.436
88	vonegrcnn_52e_full	.435
89	resnet50-SIN_IN_IN	.432
90	resnet_SIN_IN_FT_IN	.432
91	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.431
92	resnet50-VITO-8deg-cc	.430
93	cvt_cvt-21-224-in1k_4_LucyV4	.429
94	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.424
95	voneresnet-50-non_stochastic	.417
96	convnext_small_imagenet_100_seed-0	.417
97	deit_large_imagenet_full_seed-0	.416
98	resnet_50_v1_spiking	.413
99	resnet101_imagenet_full	.412
100	CLIP-RN50	.409
101	BiT-S-R50x3	.408
102	convnext_base_imagenet_full_seed-0	.406
103	voneresnet-50-robust	.403
104	xception-keras	.401
105	resnet-50-pytorch	.400
106	resnet50-sup	.400
107	resnet50_tutorial	.400
108	resnet_50_v1	.400
109	resnet50-moclr8deg	.394
110	resnet50_random_linf8_perturb	.393
111	ReAlnet03	.384
112	efficientnet_b2_imagenet_full	.383
113	resnet-50_v2-tf	.378
114	inception_v3-tf	.377
115	resnet50_random_l2_perturb	.371
116	convnext_small_imagenet_full_seed-0	.367
117	resnet-101_v1-tf	.363
118	resnet-50_v1-tf	.359
119	ReAlnet02	.355
120	resnet50-SIN_IN	.355
121	convnext_large_imagenet_full_seed-0	.354
122	Res2Net50_26w_4s	.350
123	AdvProp_efficientnet-b8	.349
124	densenet-121-keras	.346
125	resnet50	.344
126	inception_v3	.341
127	deit_small_imagenet_full_seed-0	.338
128	BiT-S-R152x2	.336
129	tv_efficientnet-b1	.336
130	nasnet_mobile	.335
131	ReAlnet04	.334
132	grcnn_109	.333
133	blt_vs	.331
134	ReAlnet07	.329
135	ReAlnet01	.328
136	xception	.327
137	ReAlnet06	.326
138	grcnn	.325
139	inception_v1	.324
140	ReAlnet08	.324
141	mobilenet_v2_1.0_192-tf	.318
142	antialiased-r50	.318
143	ReAlnet05	.317
144	inception_v2-tf	.309
145	densenet-121	.304
146	mobilenet_v2_1_0_192	.301
147	deit_base_imagenet_full_seed-0	.299
148	vonealexnet_gaussian_noise_std4_fixed	.297
149	ReAlnet10	.297
150	CORnet-S	.297
151	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.296
152	resnet18_imagenet_full	.295
153	vgg_19	.295
154	cornet_s	.293
155	resnet50_robust_l2_eps1	.291
156	alexnet	.290
157	regnet_y_400mf	.288
158	mobilenet_v1_0.75_224	.287
159	resnet50_linf_8_robust	.287
160	texture_shape_resnet50_trained_on_SIN	.286
161	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.281
162	voneresnet-50	.281
163	alexnet_ks_torevert	.281
164	alexnet-baseline	.281
165	alexnet	.281
166	alexnet	.281
167	alexnet	.281
168	alexnet	.281
169	alexnet	.281
170	alexnet	.281
171	alexnet	.281
172	alexnet	.281
173	alexnet	.281
174	ReAlnet09	.277
175	mobilenet_v2_1_0_224	.276
176	densenet-169-keras	.274
177	resnet50_l2_3_robust	.272
178	mobilenet_v1_1.0_192	.269
179	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.264
180	alexnet_random_linf8_perturb	.260
181	resnet50_moco_v2	.256
182	vgg_16	.255
183	mobilenet_v2_0_5_224	.253
184	alexnet_reduced_aliasing_early_checkpoint	.252
185	resnet50_byol	.251
186	alexnet_l2_3_robust	.250
187	efficientnet_b1_imagenet_full	.249
188	mobilenet_v2_1_3_224	.247
189	resnet-18	.245
190	resnet50-SIN	.243
191	resnet50_robust_l2_eps3	.242
192	resnet-50-robust	.242
193	imagenet_l2_3_0	.242
194	densenet-169	.241
195	mobilenet_v2_1_0_128	.239
196	BiT-S-R152x4	.238
197	mobilenet_v2_1_0_160	.235
198	mobilenet_v1_1.0_224	.230
199	mobilenet_v2_1.0_160-tf	.229
200	inception_v1-tf	.223
201	cv_18_dagger_408_pretrained	.223
202	shufflenet_v2_x1_0	.222
203	alexnet_linf_8_robust	.218
204	efficientnet_b0_imagenet_full	.217
205	BiT-S-R101x3	.213
206	resnet50-vitoimagevidnet8	.210
207	alexnet_random_l2_3_perturb	.210
208	mobilenet_v2_1.4_224-tf	.208
209	BiT-S-R50x1	.208
210	resnet-34	.208
211	mobilenet_v2_1.3_224-tf	.206
212	mobilenet_v2_0_75_160	.204
213	mobilenet_v1_1.0_160	.198
214	vgg-16-keras	.197
215	mobilenet_v2_0.5_192-tf	.196
216	resnet50_simclr	.196
217	mobilenet_v2_1.0_224-tf	.194
218	mobilenet_v1_1.0_128	.192
219	BiT-S-R101x1	.188
220	squeezenet1_1	.182
221	mobilenet_v2_0.75_96	.181
222	mobilenet_v2_0_5_192	.180
223	alexnet_training_seed_01	.180
224	alexnet_training_seed_05	.180
225	alexnet_robust_correct	.179
226	vgg-19-keras	.179
227	deit_small_imagenet_100_seed-0	.171
228	mobilenet_v2_0.75_128	.169
229	konkle_alexnetgn_ipcl_ref01_primary_model	.168
230	mobilenet_v2_0.75_160-tf	.164
231	convnext_tiny_sup	.161
232	alexnet_early_checkpoint	.161
233	mobilenet_v1_0.25_192	.160
234	0.5x_resnet-18_LC_w_sh_1_iter	.160
235	mobilenet_v2_1-4_224_pytorch	.159
236	mobilenet_v2_1_4_224	.159
237	mobilenet_v2_0.5_224-tf	.156
238	artResNet18_1	.156
239	mobilenet_v2_0.35_160	.150
240	mobilenet_v1_0.75_128	.148
241	mobilenet_v2_1.0_128-tf	.143
242	mobilenet_v1_0.75_192	.141
243	resnet50_imagenet_100_seed-0	.140
244	AdvProp_efficientnet-b0	.140
245	mobilenet_v1_0.5_128	.138
246	imagenet_l2_10_0	.132
247	nasnet_mobile-tf	.132
248	mobilenet_v2_1.0_96	.131
249	mobilenet_v2_0.35_96	.131
250	texture_shape_alexnet_trained_on_SIN	.129
251	squeezenet1_0	.126
252	mobilenet_v1_0.5_192	.126
253	mobilenet_v1_0.25_160	.125
254	AlexNet_SIN	.122
255	mobilenet_v2_0.5_160	.120
256	mobilenet_v2_0_75_192	.120
257	mobilenet_v1_0.25_224	.113
258	mobilenet_v2_0.5_128	.113
259	mobilenet_v2_0.75_192-tf	.111
260	mobilenet_v1_0.75_160	.107
261	resnet_50_v1_spiking_l4a	.098
262	mobilenet_v1_0.5_224	.097
263	mobilenet_v2_0_75_224	.097
264	resnet50-meshes-lt-100-original-scratch	.094
265	deit_small_imagenet_1_seed-0	.094
266	deit_small_imagenet_10_seed-0	.094
267	resnet50_imagenet_10_seed-0	.093
268	alexnet_training_seed_06	.089
269	alexnet_training_seed_02	.085
270	alexnet_training_seed_04	.085
271	alexnet_training_seed_08	.085
272	alexnet_training_seed_10	.085
273	convnext_small_imagenet_10_seed-0	.084
274	alexnet_training_seed_07	.084
275	mobilenet_v2_0.5_96	.077
276	resnet50_eMMCR_Vanilla	.077
277	mobilenet_v2_0.35_128	.070
278	barlow-twins-resnet50	.063
279	resnet50-vicregl0p9	.059
280	mobilenet_v2_0.75_224-tf	.058
281	resnet50-meshes-lt-100-original-pretrained	.058
282	resnet50_FractalDB	.057
283	mobilenet_v2_0.35_224	.056
284	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.055
285	resnet50-vicregl0p75	.051
286	resnet50-vicreg	.044
287	alexnet_training_seed_09	.042
288	resnet50-barlow	.041
289	mobilenet_v1_0.5_160	.038
290	mobilenet_v2_0.35_192	.036
291	my-model	.029
292	convnext_small_imagenet_1_seed-0	.015
293	resnet_50_v1_spiking_l4	.014
294	mobilenet_v1_0.25_128	.013
295	resnet50-simclr-vissl	.009
296	mobilevit_small	.008
297	resnet50_imagenet_1_seed-0	.008
298	resnet_50_v1_spiking_l3	.004
299	alexnet_training_seed_03	.002
300	resnet-50_untrained	.002
301	resnet18-instance_recognition
302	resnet-18-LC_w_sh_100_iter_conv_init_m
303	openclip
304	pnasnet_large_pytorch
305	ReAlnet01_cornet
306	resnet50
307	resnet-18-LC_w_sh_10_iter_m
308	resnet-18-LC_w_sh_1_iter_m
309	yolos_tiny
310	dorinet_cornet_z
311	pixels
312	CORnetZ_CIFAR10_bs32_20_04
313	unet_entire
314	v1-pyr-nodown
315	dcgan
316	resnet-50x2_untrained
317	yudixie_resnet18_translation_reg_0_240719
318	resnet-18_untrained
319	briaai_rmbg_1_4
320	yudixie_resnet18_distance_reg_0_240719
321	yudixie_resnet18_rotation_reg_0_240719
322	ReAlnet03_cornet
323	resnet-18-LC_untrained
324	omnivore_swinT
325	ReAlnet02_cornet
326	resnet18-contrastive_predictive
327	resnet18_ecoset_full
328	resnet34_ecoset_full
329	resnet-18-LC_w_sh_1_iter_conv_init_m
330	r3m_resnet34_nocrop
331	VOneCORnet-S
332	resnet50_ecoset_full
333	resnet101_ecoset_full
334	yudixie_resnet50_distance_rotation_0_240908
335	yudixie_resnet50_rotation_reg_0_240908
336	yudixie_resnet50_distance_translation_0_240908
337	ViT_B_16
338	resnet152_ecoset_full
339	yudixie_resnet18_distance_translation_0_240719
340	yudixie_resnet18_translation_rotation_0_240719
341	yudixie_resnet18_distance_rotation_0_240719
342	BiT-M-R152x2
343	yudixie_resnet18_category_class_0_240719
344	ReAlnet05_cornet
345	yudixie_resnet18_random_0_240719
346	resnet18-depth_prediction
347	yudixie_resnet18_cat_obj_class_all_latents_0_240719
348	resnet-18_test_m
349	my-model
350	yudixie_resnet18_distance_translation_rotation_0_240719
351	yudixie_resnet18_object_class_0_240719
352	SeLaVi-AVE
353	yudixie_resnet18_imagenet1kpret_0_240719
354	r3d_18
355	ReAlnet06_cornet
356	yudixie_resnet50_translation_reg_0_240908
357	ViT_B_32
358	ReAlnet01_ALL
359	mc3_18
360	vggface
361	yudixie_resnet50_translation_rotation_0_240908
362	ReAlnet07_cornet
363	ReAlnet02_ALL
364	s3d
365	ViT_L_32
366	mvit_v1_b
367	ReAlnet04_ALL
368	ReAlnet08_cornet
369	resnet18-simclr
370	ReAlnet03_ALL
371	resnet18-autoencoder
372	resnet18-relative_position
373	omnivore_swinB
374	SeLaVi-Kinetics400
375	ReAlnet09_cornet
376	ReAlnet05_ALL
377	SeLaVi-VGG-Sound
378	BiT-M-R50x1
379	RN50
380	BiT-M-R50x3
381	0.5x_resnet-18_LC_w_sh_100_iter
382	SeLaVi-Kinetics-Sound
383	ReAlnet06_ALL
384	RN50
385	omnivore_swinS
386	ReAlnet07_ALL
387	mvit_v2_s
388	r2plus1d_18
389	resnet18-colorization
390	0.5x_resnet-18
391	resnet18-contrastive_multiview
392	resnet-18-LC_1st_conv_m
393	ReAlnet10_cornet
394	resnet18-supervised
395	yudixie_resnet50_distance_translation_rotation_0_240908
396	ReAlnet08_ALL
397	ReAlnet09_ALL
398	resnet-18-LC_m
399	r3m_resnet50_nocrop
400	r3m_resnet34
401	ReAlnet04_cornet
402	ViT-B/32
403	resnet50_primary_visual_cortex
404	ReAlnet10_ALL
405	0.5x_resnet-18_LC_w_sh_10_iter
406	FrankRobWobv0
407	resnet-18-LC_conv_init_m
408	yudixie_resnet50_object_class_0_240908
409	yudixie_resnet50_cat_obj_class_all_latents_0_240908
410	resnet-18-LC_w_sh_10_iter_conv_init_m
411	yudixie_resnet50_imagenet1kpret_0_240312
412	resnet18-local_aggregation
413	yudixie_resnet50_category_class_0_240908
414	yudixie_resnet50_imagenet1kpret_0_240908
415	0.5x_resnet-18_LC
416	CORnetZ_CIFAR10
417	yudixie_resnet50_random_0_240908
418	resnet18-deepcluster
419	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
420	resnet50_ImageNet
421	resnet50_pretrained_with_retinal_waves
422	r3m_resnet50
423	resnet50-cifar
424	resnet18_imagenet21kP
425	resnet-18-LC_w_sh_100_iter_m
426	yudixie_resnet50_distance_reg_0_240908
427	ConvLSTM
428	PredRNN
429	SimVP
430	TAU
431	MIM
432	hmax
433	r3m_resnet18
434	CORnet-Z

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.39.

Note that scores are relative to this ceiling.

Data: Geirhos2021eidolonI

Metric: error_consistency