Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021rotation-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.646
2	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.614
3	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.606
4	resnet-152_v2_pytorch	.589
5	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.581
6	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.571
7	fixres_resnext101_32x48d_wsl	.568
8	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.565
9	resnet_152_v2	.561
10	vit_large_patch14_clip_336:laion2b_ft_in1k	.559
11	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.546
12	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.539
13	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.528
14	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.527
15	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.516
16	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.506
17	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.500
18	effnetv2m_custom384	.489
19	vit_large_patch14_clip_224:openai_ft_in1k	.488
20	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.478
21	convnext_base:clip_laiona_augreg_ft_in1k_384	.476
22	convnext_large:fb_in22k_ft_in1k	.467
23	vit_large_patch14_clip_224:laion2b_ft_in1k	.457
24	pnasnet_large	.431
25	vit_base_patch16_clip_224:openai_ft_in1k	.429
26	custom_model_cv_18_dagger_408	.425
27	convnext_xlarge:fb_in22k_ft_in1k	.417
28	resnet_101_v2	.416
29	resnet_50_v2	.413
30	resnext101_32x48d_wsl	.405
31	CLIP_resnet50	.404
32	CLIP_resnet50_float32	.404
33	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.404
34	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.400
35	resnext101_32x16d_wsl	.391
36	resnext101_32x32d_wsl	.385
37	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.380
38	SWSL_resnet50	.370
39	SWSL_resnext101_32x8d	.368
40	CLIP-RN50	.367
41	pnasnet_large-tf	.357
42	effnetb1_272x240	.356
43	resnext101_32x8d_wsl	.351
44	nasnet_large	.344
45	cvt_cvt-13-384-in1k_4_LucyV4	.343
46	cvt_cvt-21-384-in1k_4_LucyV4	.338
47	convnext_tiny:in12k_ft_in1k	.334
48	nasnet_large-tf	.325
49	vision_transformer_vit_large_patch16_224	.308
50	antialias-resnet152	.282
51	inception_v3	.273
52	CLIP_ViT-B_32	.273
53	focalnet_tiny_lrf_in1k	.273
54	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.268
55	inception_v4	.266
56	inception_v3-tf	.264
57	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.261
58	resnet152_imagenet_full	.259
59	convnext_femto_ols:d1_in1k	.255
60	resnet101	.255
61	resnet50-VITO-8deg-cc	.254
62	resnet101_imagenet_full	.250
63	vit_relpos_base_patch16_clsgap_224:sw_in1k	.249
64	resnet-152_v2-tf	.239
65	convnext_large_imagenet_full_seed-0	.236
66	resnet_152_v1	.236
67	cv_18_dagger_408_pretrained	.226
68	xception-keras	.225
69	antialiased-rnext101_32x8d	.225
70	convnext_tiny_imagenet_full_seed-0	.224
71	resnet50_random_l2_perturb	.223
72	tv_efficientnet-b1	.221
73	cvt_cvt-13-224-in1k_4_LucyV4	.218
74	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.213
75	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.212
76	convnext_base_imagenet_full_seed-0	.206
77	resnet50	.206
78	inception_v4-tf	.204
79	resnet50_imagenet_full	.204
80	deit_base_imagenet_full_seed-0	.201
81	convnext_small_imagenet_full_seed-0	.197
82	deit_large_imagenet_full_seed-0	.196
83	resnet_101_v1	.195
84	antialiased-r50	.195
85	grcnn_109	.194
86	resnet50-moclr8deg	.190
87	deit_small_imagenet_full_seed-0	.190
88	ViT_B_16_imagenet1k	.182
89	cvt_cvt-21-224-in1k_4_LucyV4	.181
90	densenet-201-keras	.181
91	xception	.179
92	Res2Net50_26w_4s	.177
93	pnasnet_large_pytorch	.175
94	densenet_201_pytorch	.172
95	densenet-201	.172
96	resnet_50_v1_spiking	.172
97	efficientnet_b2_imagenet_full	.167
98	vgg_19	.166
99	efficientnet_b0	.165
100	densenet-121-keras	.164
101	convnext_tiny_sup	.161
102	ViT_L_32_imagenet1k	.161
103	grcnn	.161
104	resnet34_imagenet_full	.160
105	resnet-50_v2-tf	.156
106	densenet-121	.155
107	resnet50_tutorial	.154
108	resnet_50_v1	.154
109	resnet50-sup	.154
110	resnet-50-pytorch	.154
111	mobilenet_v2_1.4_224-tf	.153
112	resnet50_random_linf8_perturb	.152
113	ViT_B_32_imagenet1k	.152
114	AdvProp_efficientnet-b6	.150
115	resnet_SIN_IN_FT_IN	.150
116	resnet50-SIN_IN_IN	.150
117	BiT-S-R50x3	.149
118	inception_v2-tf	.149
119	resnet-101_v1-tf	.149
120	resnet50_byol	.144
121	efficientnet-b7	.143
122	nasnet_mobile	.143
123	mobilenet_v2_1_0_224	.140
124	AT_efficientnet-b2	.137
125	AdvProp_efficientnet-b2	.137
126	resnet50-vitoimagevidnet8	.135
127	cornet_s	.133
128	BiT-S-R152x2	.132
129	densenet-169-keras	.131
130	voneresnet-50-non_stochastic	.130
131	BiT-S-R101x3	.130
132	mobilenet_v1_0.75_224	.130
133	mobilenet_v2_1.0_224-tf	.130
134	resnet18_imagenet_full	.130
135	efficientnet_b0_imagenet_full	.129
136	mobilenet_v2_1_3_224	.124
137	ReAlnet04	.124
138	AdvProp_efficientnet-b7	.123
139	inception_v1	.123
140	nasnet_mobile-tf	.121
141	ReAlnet03	.120
142	resnet50-SIN_IN	.119
143	resnet-50_v1-tf	.118
144	convnext_small_imagenet_100_seed-0	.117
145	ReAlnet10	.116
146	resnet_50_v1_spiking_l4a	.116
147	vgg_16	.116
148	AdvProp_efficientnet-b4	.115
149	AT_efficientnet-b4	.115
150	ReAlnet06	.115
151	ReAlnet01	.114
152	ReAlnet02	.114
153	ReAlnet09	.114
154	resnet-152_v1-tf	.114
155	densenet-169	.112
156	mobilenet_v2_1.3_224-tf	.109
157	resnet50_moco_v2	.105
158	BiT-S-R152x4	.105
159	efficientnet_b1_imagenet_full	.101
160	mobilenet_v1_1.0_192	.100
161	inception_v1-tf	.098
162	ReAlnet05	.098
163	mobilenet_v2_1.0_160-tf	.096
164	vonegrcnn_47e	.096
165	ReAlnet07	.095
166	mobilenet_v2_1_0_192	.094
167	CORnet-S	.094
168	mobilenet_v1_1.0_224	.092
169	ReAlnet08	.092
170	mobilenet_v2_1_0_160	.091
171	mobilenet_v2_0_5_224	.091
172	resnet-34	.090
173	mobilenet_v2_0_75_192	.089
174	BiT-S-R50x1	.089
175	mobilenet_v2_0_75_160	.088
176	mobilenet_v2_1_4_224	.087
177	mobilenet_v2_1-4_224_pytorch	.087
178	resnet50_linf_4_robust	.085
179	squeezenet1_1	.085
180	BiT-S-R101x1	.085
181	vonegrcnn_62e_nobn	.085
182	regnet_y_400mf	.085
183	vgg-16-keras	.084
184	vonegrcnn_47e	.081
185	resnet50_simclr	.081
186	mobilenet_v2_0_75_224	.080
187	vonegrcnn_52e_full	.080
188	resnet-18	.079
189	vonegrcnn_52e_full	.079
190	mobilenet_v2_0_5_192	.079
191	AdvProp_efficientnet-b0	.078
192	vonegrcnn_62e_nobn	.075
193	shufflenet_v2_x1_0	.074
194	resnet50_l2_3_robust	.073
195	mobilenet_v1_0.5_192	.073
196	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.073
197	0.5x_resnet-18_LC_w_sh_1_iter	.070
198	mobilenet_v2_0.75_160-tf	.070
199	alexnet	.069
200	mobilenet_v2_1.0_192-tf	.069
201	texture_shape_alexnet_trained_on_SIN	.068
202	mobilenet_v2_0.75_224-tf	.067
203	mobilenet_v2_0.75_192-tf	.064
204	texture_shape_resnet50_trained_on_SIN	.064
205	mobilenet_v2_0.35_192	.064
206	mobilenet_v2_1_0_128	.063
207	mobilenet_v2_0.75_96	.063
208	mobilenet_v1_0.25_224	.062
209	squeezenet1_0	.062
210	resnet50_robust_l2_eps1	.062
211	resnet50_linf_8_robust	.061
212	mobilenet_v2_0.5_192-tf	.061
213	mobilenet_v1_1.0_128	.061
214	AdvProp_efficientnet-b8	.061
215	mobilenet_v1_0.25_192	.060
216	mobilenet_v1_0.75_160	.060
217	mobilenet_v2_0.35_224	.059
218	blt_vs	.059
219	konkle_alexnetgn_ipcl_ref01_primary_model	.059
220	mobilenet_v1_0.5_224	.058
221	resnet50_imagenet_100_seed-0	.058
222	alexnet_random_linf8_perturb	.058
223	resnet50_robust_l2_eps3	.058
224	resnet-50-robust	.058
225	imagenet_l2_3_0	.058
226	voneresnet-50	.057
227	vgg-19-keras	.057
228	alexnet_reduced_aliasing_early_checkpoint	.056
229	mobilenet_v2_0.5_224-tf	.055
230	mobilenet_v2_0.75_128	.055
231	mobilenet_v2_0.35_96	.055
232	resnet50-SIN	.055
233	convnext_small_imagenet_1_seed-0	.054
234	mobilenet_v2_1.0_128-tf	.054
235	mobilenet_v1_0.75_128	.053
236	AlexNet_SIN	.052
237	mobilenet_v2_0.35_160	.052
238	mobilenet_v1_0.75_192	.052
239	voneresnet-50-robust	.052
240	mobilenet_v1_1.0_160	.050
241	alexnet_random_l2_3_perturb	.050
242	mobilenet_v2_0.5_96	.050
243	mobilenet_v2_0.5_160	.049
244	alexnet_ks_torevert	.049
245	alexnet	.049
246	alexnet	.049
247	alexnet	.049
248	alexnet	.049
249	alexnet	.049
250	alexnet	.049
251	alexnet	.049
252	alexnet	.049
253	alexnet	.049
254	alexnet-baseline	.049
255	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.049
256	mobilenet_v2_0.35_128	.048
257	mobilenet_v1_0.5_128	.046
258	vonealexnet_gaussian_noise_std4_fixed	.045
259	alexnet_early_checkpoint	.044
260	alexnet_l2_3_robust	.044
261	resnet_50_v1_spiking_l4	.042
262	mobilenet_v2_1.0_96	.042
263	mobilenet_v2_0.5_128	.038
264	alexnet_training_seed_05	.037
265	alexnet_training_seed_01	.037
266	mobilenet_v1_0.25_160	.035
267	imagenet_l2_10_0	.035
268	mobilenet_v1_0.5_160	.033
269	resnet50-vicregl0p75	.032
270	alexnet_linf_8_robust	.029
271	alexnet_robust_correct	.023
272	resnet-50_untrained	.021
273	alexnet_training_seed_06	.018
274	deit_small_imagenet_100_seed-0	.018
275	resnet50-meshes-lt-100-original-pretrained	.018
276	alexnet_training_seed_03	.016
277	resnet50-vicreg	.015
278	resnet50_FractalDB	.012
279	artResNet18_1	.011
280	alexnet_training_seed_07	.011
281	resnet50-vicregl0p9	.009
282	alexnet_training_seed_04	.009
283	alexnet_training_seed_02	.009
284	alexnet_training_seed_08	.009
285	alexnet_training_seed_10	.009
286	resnet50-meshes-lt-100-original-scratch	.009
287	mobilenet_v1_0.25_128	.008
288	deit_small_imagenet_1_seed-0	.008
289	deit_small_imagenet_10_seed-0	.008
290	convnext_small_imagenet_10_seed-0	.007
291	resnet50_imagenet_10_seed-0	.007
292	mobilevit_small	.007
293	resnet50_imagenet_1_seed-0	.006
294	resnet50-barlow	.004
295	resnet50-simclr-vissl	.003
296	barlow-twins-resnet50	.002
297	resnet_50_v1_spiking_l3	.001
298	alexnet_training_seed_09	.001
299	yudixie_resnet18_distance_translation_rotation_0_240719
300	yudixie_resnet18_object_class_0_240719
301	yudixie_resnet18_imagenet1kpret_0_240719
302	RN50
303	ReAlnet06_cornet
304	yudixie_resnet50_translation_reg_0_240908
305	yudixie_resnet50_translation_rotation_0_240908
306	BiT-M-R50x1
307	BiT-M-R50x3
308	ReAlnet07_cornet
309	resnet18-colorization
310	ReAlnet08_cornet
311	0.5x_resnet-18
312	VOneCORnet-S
313	omnivore_swinB
314	ReAlnet09_cornet
315	resnet-18-LC_conv_init_m
316	ViT-B/32
317	omnivore_swinS
318	resnet-18-LC_1st_conv_m
319	resnet18-supervised
320	r3m_resnet50_nocrop
321	ReAlnet10_cornet
322	resnet-18-LC_w_sh_100_iter_conv_init_m
323	resnet-18-LC_m
324	r3m_resnet34
325	my-model
326	r3m_resnet50
327	yudixie_resnet50_distance_translation_rotation_0_240908
328	resnet50_primary_visual_cortex
329	0.5x_resnet-18_LC_w_sh_10_iter
330	resnet18-contrastive_multiview
331	resnet50-cifar
332	resnet50_pretrained_with_retinal_waves
333	resnet50_ImageNet
334	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
335	resnet18-deepcluster
336	resnet18-local_aggregation
337	CORnetZ_CIFAR10
338	0.5x_resnet-18_LC
339	r3m_resnet18
340	CORnet-Z
341	resnet-18-LC_w_sh_1_iter_m
342	FrankRobWobv0
343	yudixie_resnet50_imagenet1kpret_0_240312
344	yudixie_resnet50_imagenet1kpret_0_240908
345	yudixie_resnet50_random_0_240908
346	yudixie_resnet50_cat_obj_class_all_latents_0_240908
347	yudixie_resnet50_object_class_0_240908
348	resnet-18-LC_w_sh_10_iter_conv_init_m
349	resnet-18-LC_w_sh_100_iter_m
350	openclip
351	resnet50
352	yudixie_resnet50_category_class_0_240908
353	my-model
354	dorinet_cornet_z
355	pixels
356	omnivore_swinT
357	CORnetZ_CIFAR10_bs32_20_04
358	v1-pyr-nodown
359	resnet18_imagenet21kP
360	ConvLSTM
361	PredRNN
362	yudixie_resnet50_distance_reg_0_240908
363	SimVP
364	unet_entire
365	TAU
366	dcgan
367	MIM
368	hmax
369	resnet18-instance_recognition
370	resnet-50x2_untrained
371	resnet-18-LC_w_sh_10_iter_m
372	resnet-18_untrained
373	resnet-18-LC_untrained
374	inception_v3_pytorch
375	resnet18-contrastive_predictive
376	r3m_resnet34_nocrop
377	yolos_tiny
378	ViT_B_16
379	resnet18-depth_prediction
380	ReAlnet01_cornet
381	ViT_B_32
382	ViT_L_32
383	vggface
384	yudixie_resnet18_translation_reg_0_240719
385	yudixie_resnet18_distance_reg_0_240719
386	yudixie_resnet18_rotation_reg_0_240719
387	briaai_rmbg_1_4
388	resnet18-simclr
389	ReAlnet03_cornet
390	resnet-18-LC_w_sh_1_iter_conv_init_m
391	resnet18-autoencoder
392	resnet18_ecoset_full
393	resnet18-relative_position
394	BiT-M-R152x2
395	resnet34_ecoset_full
396	ReAlnet02_cornet
397	resnet50_ecoset_full
398	resnet101_ecoset_full
399	yudixie_resnet50_distance_translation_0_240908
400	yudixie_resnet50_rotation_reg_0_240908
401	resnet152_ecoset_full
402	yudixie_resnet18_distance_translation_0_240719
403	ReAlnet04_cornet
404	yudixie_resnet18_translation_rotation_0_240719
405	yudixie_resnet18_distance_rotation_0_240719
406	yudixie_resnet50_distance_rotation_0_240908
407	yudixie_resnet18_category_class_0_240719
408	ReAlnet05_cornet
409	yudixie_resnet18_random_0_240719
410	yudixie_resnet18_cat_obj_class_all_latents_0_240719
411	resnet-18_test_m
412	RN50
413	0.5x_resnet-18_LC_w_sh_100_iter

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.44.

Note that scores are relative to this ceiling.

Data: Geirhos2021rotation

Metric: error_consistency