Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021stylized-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	resnext101_32x32d_wsl	.871
2	resnext101_32x16d_wsl	.859
3	resnext101_32x48d_wsl	.851
4	fixres_resnext101_32x48d_wsl	.830
5	SWSL_resnext101_32x8d	.787
6	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.758
7	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.751
8	convnext_base:clip_laiona_augreg_ft_in1k_384	.750
9	convnext_xlarge:fb_in22k_ft_in1k	.746
10	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.737
11	vit_large_patch14_clip_336:laion2b_ft_in1k	.734
12	vit_large_patch14_clip_224:laion2b_ft_in1k	.723
13	vit_base_patch16_clip_224:openai_ft_in1k	.708
14	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.706
15	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.705
16	vision_transformer_vit_large_patch16_224	.705
17	convnext_large:fb_in22k_ft_in1k	.704
18	SWSL_resnet50	.702
19	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.697
20	vit_large_patch14_clip_224:openai_ft_in1k	.697
21	effnetb1_272x240	.693
22	effnetv2m_custom384	.680
23	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.680
24	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.678
25	cvt_cvt-13-384-in1k_4_LucyV4	.678
26	resnext101_32x8d_wsl	.666
27	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.654
28	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.653
29	CLIP_resnet50_float32	.646
30	CLIP_resnet50	.646
31	nasnet_large	.646
32	cvt_cvt-21-384-in1k_4_LucyV4	.646
33	CLIP_ViT-B_32	.628
34	pnasnet_large	.627
35	resnet_101_v2	.623
36	resnet_152_v2	.603
37	cv_18_dagger_408_pretrained	.600
38	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.590
39	resnet-152_v2-tf	.588
40	nasnet_large-tf	.583
41	pnasnet_large-tf	.579
42	ViT_B_16_imagenet1k	.570
43	resnet_50_v2	.555
44	ViT_L_32_imagenet1k	.538
45	vit_relpos_base_patch16_clsgap_224:sw_in1k	.516
46	xception-keras	.515
47	CLIP-RN50	.511
48	ViT_B_32_imagenet1k	.494
49	inception_v4	.488
50	antialias-resnet152	.467
51	AdvProp_efficientnet-b6	.463
52	resnet152_imagenet_full	.457
53	resnet_152_v1	.456
54	xception	.450
55	resnet50-VITO-8deg-cc	.449
56	AdvProp_efficientnet-b4	.443
57	AT_efficientnet-b4	.443
58	resnet-50_v2-tf	.439
59	resnet50_imagenet_full	.438
60	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.438
61	convnext_base_imagenet_full_seed-0	.433
62	convnext_small_imagenet_full_seed-0	.429
63	AdvProp_efficientnet-b7	.427
64	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.418
65	cvt_cvt-13-224-in1k_4_LucyV4	.417
66	convnext_large_imagenet_full_seed-0	.417
67	resnet101	.417
68	inception_v4-tf	.411
69	convnext_femto_ols:d1_in1k	.409
70	inception_v3	.405
71	resnet101_imagenet_full	.405
72	convnext_tiny_imagenet_full_seed-0	.395
73	efficientnet-b7	.392
74	inception_v3-tf	.391
75	resnet50-SIN_IN	.389
76	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.381
77	grcnn_109	.370
78	efficientnet_b0	.369
79	densenet-201-keras	.365
80	resnet50	.362
81	resnet_101_v1	.358
82	antialiased-rnext101_32x8d	.358
83	resnet50_random_l2_perturb	.351
84	focalnet_tiny_lrf_in1k	.346
85	densenet-201	.342
86	grcnn	.340
87	tv_efficientnet-b1	.324
88	densenet-121	.322
89	texture_shape_resnet50_trained_on_SIN	.321
90	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.320
91	densenet-121-keras	.319
92	custom_model_cv_18_dagger_408	.319
93	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.317
94	ReAlnet01	.315
95	ReAlnet03	.315
96	AT_efficientnet-b2	.315
97	AdvProp_efficientnet-b2	.315
98	deit_large_imagenet_full_seed-0	.314
99	resnet50_byol	.314
100	Res2Net50_26w_4s	.313
101	cvt_cvt-21-224-in1k_4_LucyV4	.311
102	ReAlnet09	.307
103	ReAlnet02	.305
104	AdvProp_efficientnet-b8	.303
105	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.302
106	voneresnet-50-non_stochastic	.301
107	ReAlnet04	.300
108	resnet34_imagenet_full	.299
109	cornet_s	.299
110	inception_v1	.298
111	ReAlnet06	.297
112	convnext_tiny:in12k_ft_in1k	.295
113	resnet50-moclr8deg	.295
114	BiT-S-R152x4	.293
115	resnet18_imagenet_full	.292
116	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.290
117	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.289
118	mobilenet_v2_1.4_224-tf	.288
119	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.281
120	antialiased-r50	.281
121	mobilenet_v2_1_0_224	.278
122	resnet50-SIN_IN_IN	.274
123	resnet_SIN_IN_FT_IN	.274
124	densenet-169	.273
125	resnet50_moco_v2	.272
126	ReAlnet08	.271
127	inception_v2-tf	.270
128	ReAlnet07	.267
129	inception_v1-tf	.266
130	vgg_19	.263
131	ReAlnet10	.263
132	densenet-169-keras	.262
133	resnet50_random_linf8_perturb	.262
134	vonegrcnn_47e	.258
135	nasnet_mobile	.255
136	resnet_50_v1	.254
137	resnet50_tutorial	.254
138	resnet50-sup	.254
139	resnet-50-pytorch	.254
140	resnet50-vitoimagevidnet8	.252
141	ReAlnet05	.251
142	BiT-S-R101x3	.251
143	CORnet-S	.236
144	resnet_50_v1_spiking	.233
145	resnet-50_v1-tf	.233
146	vonegrcnn_62e_nobn	.232
147	BiT-S-R50x3	.230
148	resnet-34	.229
149	vonegrcnn_52e_full	.229
150	resnet50_simclr	.224
151	BiT-S-R152x2	.223
152	voneresnet-50-robust	.219
153	BiT-S-R50x1	.218
154	mobilenet_v2_1_3_224	.216
155	vonegrcnn_62e_nobn	.213
156	mobilenet_v2_1_0_192	.208
157	vonegrcnn_47e	.202
158	efficientnet_b2_imagenet_full	.202
159	mobilenet_v2_1.0_224-tf	.201
160	resnet-18	.199
161	alexnet_training_seed_04	.198
162	alexnet_training_seed_02	.198
163	alexnet_training_seed_08	.198
164	alexnet_training_seed_10	.198
165	resnet-101_v1-tf	.197
166	resnet-152_v1-tf	.195
167	resnet50_linf_4_robust	.195
168	BiT-S-R101x1	.194
169	artResNet18_1	.192
170	vgg_16	.192
171	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.191
172	resnet50-SIN	.187
173	deit_base_imagenet_full_seed-0	.185
174	nasnet_mobile-tf	.185
175	vonegrcnn_52e_full	.184
176	vonealexnet_gaussian_noise_std4_fixed	.182
177	mobilenet_v2_1-4_224_pytorch	.181
178	mobilenet_v2_1_4_224	.181
179	mobilenet_v2_1.3_224-tf	.176
180	alexnet_training_seed_07	.166
181	resnet50-simclr-vissl	.166
182	resnet50_robust_l2_eps1	.163
183	mobilenet_v2_1_0_160	.160
184	efficientnet_b1_imagenet_full	.159
185	resnet50_imagenet_100_seed-0	.157
186	mobilenet_v1_0.75_224	.155
187	voneresnet-50	.154
188	resnet50-vicregl0p9	.146
189	mobilenet_v2_0_5_224	.145
190	AlexNet_SIN	.141
191	alexnet_ks_torevert	.139
192	alexnet-baseline	.139
193	alexnet	.139
194	alexnet	.139
195	alexnet	.139
196	alexnet	.139
197	alexnet	.139
198	alexnet	.139
199	alexnet	.139
200	alexnet	.139
201	alexnet	.139
202	vgg-19-keras	.134
203	texture_shape_alexnet_trained_on_SIN	.133
204	alexnet	.132
205	alexnet_linf_8_robust	.131
206	deit_small_imagenet_full_seed-0	.131
207	blt_vs	.129
208	resnet-50_untrained	.127
209	resnet50-vicregl0p75	.127
210	mobilenet_v2_0.75_128	.123
211	resnet50_l2_3_robust	.121
212	regnet_y_400mf	.121
213	efficientnet_b0_imagenet_full	.120
214	mobilenet_v1_1.0_192	.119
215	resnet50-barlow	.119
216	mobilenet_v2_0_75_192	.116
217	AdvProp_efficientnet-b0	.114
218	alexnet_random_linf8_perturb	.113
219	mobilenet_v2_0.75_224-tf	.111
220	mobilenet_v1_0.75_192	.110
221	mobilenet_v2_0.5_192-tf	.110
222	alexnet_robust_correct	.107
223	alexnet_random_l2_3_perturb	.107
224	mobilenet_v2_1_0_128	.106
225	mobilenet_v1_0.5_192	.106
226	alexnet_l2_3_robust	.103
227	mobilenet_v1_1.0_224	.103
228	vgg-16-keras	.102
229	resnet50_imagenet_10_seed-0	.100
230	mobilenet_v2_1.0_192-tf	.098
231	mobilenet_v2_0_75_160	.097
232	konkle_alexnetgn_ipcl_ref01_primary_model	.097
233	alexnet_reduced_aliasing_early_checkpoint	.097
234	imagenet_l2_10_0	.097
235	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.096
236	resnet50_linf_8_robust	.094
237	mobilenet_v1_1.0_160	.094
238	alexnet_training_seed_01	.093
239	alexnet_training_seed_05	.093
240	mobilenet_v2_1.0_160-tf	.092
241	mobilenet_v2_0.5_224-tf	.090
242	mobilenet_v2_0_75_224	.089
243	convnext_small_imagenet_1_seed-0	.088
244	mobilenet_v2_0.35_96	.088
245	mobilenet_v1_0.25_128	.087
246	mobilenet_v2_0_5_192	.087
247	resnet50_imagenet_1_seed-0	.086
248	imagenet_l2_3_0	.084
249	resnet50_robust_l2_eps3	.084
250	resnet-50-robust	.084
251	convnext_tiny_sup	.080
252	mobilenet_v2_0.35_224	.079
253	mobilenet_v2_0.35_160	.078
254	mobilenet_v2_1.0_96	.078
255	resnet_50_v1_spiking_l3	.075
256	mobilenet_v2_0.75_96	.074
257	alexnet_early_checkpoint	.070
258	squeezenet1_1	.066
259	mobilenet_v2_0.5_128	.065
260	convnext_small_imagenet_100_seed-0	.064
261	mobilenet_v2_0.75_192-tf	.064
262	mobilevit_small	.064
263	mobilenet_v2_0.75_160-tf	.062
264	mobilenet_v2_0.5_96	.061
265	mobilenet_v1_0.25_224	.060
266	resnet50_FractalDB	.057
267	deit_small_imagenet_100_seed-0	.057
268	mobilenet_v2_0.5_160	.049
269	deit_small_imagenet_1_seed-0	.047
270	shufflenet_v2_x1_0	.046
271	alexnet_training_seed_09	.045
272	deit_small_imagenet_10_seed-0	.045
273	resnet50-meshes-lt-100-original-pretrained	.044
274	alexnet_training_seed_03	.044
275	resnet50-meshes-lt-100-original-scratch	.044
276	mobilenet_v2_0.35_192	.040
277	mobilenet_v1_0.25_160	.040
278	convnext_small_imagenet_10_seed-0	.040
279	mobilenet_v1_1.0_128	.038
280	mobilenet_v2_1.0_128-tf	.037
281	mobilenet_v1_0.5_160	.028
282	squeezenet1_0	.026
283	mobilenet_v2_0.35_128	.026
284	0.5x_resnet-18_LC_w_sh_1_iter	.025
285	resnet_50_v1_spiking_l4	.016
286	resnet50-vicreg	.016
287	mobilenet_v1_0.25_192	.015
288	mobilenet_v1_0.5_128	.015
289	mobilenet_v1_0.5_224	.015
290	mobilenet_v1_0.75_160	.014
291	barlow-twins-resnet50	.012
292	mobilenet_v1_0.75_128	.012
293	alexnet_training_seed_06	.008
294	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.006
295	resnet_50_v1_spiking_l4a	.005
296	resnet18_ecoset_full
297	ReAlnet02_cornet
298	0.5x_resnet-18_LC_w_sh_10_iter
299	resnet34_ecoset_full
300	VOneCORnet-S
301	resnet50_ecoset_full
302	resnet18-instance_recognition
303	ReAlnet04_cornet
304	resnet152_ecoset_full
305	my-model
306	vggface
307	yudixie_resnet18_distance_translation_0_240719
308	yudixie_resnet50_distance_translation_0_240908
309	yudixie_resnet18_translation_rotation_0_240719
310	yudixie_resnet18_distance_rotation_0_240719
311	yudixie_resnet18_category_class_0_240719
312	ReAlnet05_cornet
313	yudixie_resnet50_distance_rotation_0_240908
314	yudixie_resnet18_random_0_240719
315	yudixie_resnet18_cat_obj_class_all_latents_0_240719
316	resnet-18_test_m
317	yudixie_resnet18_distance_translation_rotation_0_240719
318	yudixie_resnet18_object_class_0_240719
319	ReAlnet06_cornet
320	yudixie_resnet50_translation_reg_0_240908
321	yudixie_resnet50_rotation_reg_0_240908
322	yudixie_resnet50_translation_rotation_0_240908
323	ReAlnet07_cornet
324	CORnet-Z
325	r3m_resnet18
326	ReAlnet08_cornet
327	resnet18-deepcluster
328	resnet18-local_aggregation
329	omnivore_swinB
330	CORnetZ_CIFAR10
331	0.5x_resnet-18_LC
332	FrankRobWobv0
333	resnet-18-LC_w_sh_10_iter_conv_init_m
334	ReAlnet09_cornet
335	resnet-18-LC_w_sh_100_iter_m
336	resnet-18-LC_w_sh_10_iter_m
337	omnivore_swinS
338	yudixie_resnet50_distance_translation_rotation_0_240908
339	resnet-18-LC_w_sh_1_iter_m
340	ReAlnet10_cornet
341	resnet-18-LC_w_sh_1_iter_conv_init_m
342	BiT-M-R152x2
343	BiT-M-R50x1
344	BiT-M-R50x3
345	resnet50_primary_visual_cortex
346	ViT-B/32
347	resnet-18-LC_w_sh_100_iter_conv_init_m
348	resnet101_ecoset_full
349	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
350	resnet50_ImageNet
351	resnet50_pretrained_with_retinal_waves
352	resnet50-cifar
353	yudixie_resnet50_imagenet1kpret_0_240908
354	yudixie_resnet50_random_0_240908
355	yudixie_resnet50_cat_obj_class_all_latents_0_240908
356	openclip
357	resnet50
358	yudixie_resnet50_imagenet1kpret_0_240312
359	my-model
360	dorinet_cornet_z
361	pixels
362	CORnetZ_CIFAR10_bs32_20_04
363	v1-pyr-nodown
364	unet_entire
365	yudixie_resnet50_object_class_0_240908
366	dcgan
367	resnet-50x2_untrained
368	resnet-18_untrained
369	resnet-18-LC_untrained
370	resnet18-contrastive_predictive
371	hmax
372	r3m_resnet34_nocrop
373	ViT_B_16
374	resnet18-depth_prediction
375	ViT_B_32
376	omnivore_swinT
377	ViT_L_32
378	resnet18-simclr
379	resnet18-autoencoder
380	yudixie_resnet50_category_class_0_240908
381	resnet18-relative_position
382	resnet18-colorization
383	RN50
384	0.5x_resnet-18_LC_w_sh_100_iter
385	yudixie_resnet50_distance_reg_0_240908
386	ConvLSTM
387	PredRNN
388	SimVP
389	TAU
390	RN50
391	MIM
392	resnet-152_v2_pytorch
393	briaai_rmbg_1_4
394	densenet_201_pytorch
395	pnasnet_large_pytorch
396	inception_v3_pytorch
397	0.5x_resnet-18
398	ReAlnet01_cornet
399	resnet18-contrastive_multiview
400	resnet-18-LC_conv_init_m
401	resnet-18-LC_1st_conv_m
402	resnet18-supervised
403	r3m_resnet50_nocrop
404	resnet-18-LC_m
405	r3m_resnet34
406	yudixie_resnet18_translation_reg_0_240719
407	yudixie_resnet18_distance_reg_0_240719
408	yolos_tiny
409	yudixie_resnet18_rotation_reg_0_240719
410	ReAlnet03_cornet
411	yudixie_resnet18_imagenet1kpret_0_240719
412	r3m_resnet50

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.50.

Note that scores are relative to this ceiling.

Data: Geirhos2021stylized

Metric: error_consistency