Leaderboard Explore Compare Profile Community Blog FAQ Competition 2024 Report Issue

Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021sketch-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.776
2	vit_large_patch14_clip_224:openai_ft_in1k	.776
3	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.746
4	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.742
5	vit_large_patch14_clip_336:laion2b_ft_in1k	.734
6	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.700
7	SWSL_resnext101_32x8d	.699
8	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.678
9	vit_large_patch14_clip_224:laion2b_ft_in1k	.669
10	CLIP_resnet50	.666
11	CLIP_resnet50_float32	.666
12	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.663
13	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.658
14	convnext_base:clip_laiona_augreg_ft_in1k_384	.658
15	CLIP-RN50	.630
16	convnext_xlarge:fb_in22k_ft_in1k	.594
17	vit_base_patch16_clip_224:openai_ft_in1k	.585
18	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.578
19	fixres_resnext101_32x48d_wsl	.576
20	CLIP_ViT-B_32	.559
21	convnext_large:fb_in22k_ft_in1k	.556
22	resnext101_32x16d_wsl	.539
23	resnext101_32x48d_wsl	.512
24	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.481
25	SWSL_resnet50	.461
26	resnext101_32x32d_wsl	.460
27	effnetv2m_custom384	.452
28	resnext101_32x8d_wsl	.425
29	vision_transformer_vit_large_patch16_224	.421
30	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.382
31	convnext_tiny:in12k_ft_in1k	.336
32	cvt_cvt-13-384-in1k_4_LucyV4	.331
33	vit_relpos_base_patch16_clsgap_224:sw_in1k	.314
34	cvt_cvt-21-384-in1k_4_LucyV4	.293
35	convnext_large_imagenet_full_seed-0	.287
36	effnetb1_272x240	.283
37	convnext_small_imagenet_full_seed-0	.278
38	convnext_tiny_imagenet_full_seed-0	.276
39	convnext_base_imagenet_full_seed-0	.274
40	focalnet_tiny_lrf_in1k	.274
41	cv_18_dagger_408_pretrained	.271
42	pnasnet_large	.267
43	resnet_152_v2	.253
44	convnext_femto_ols:d1_in1k	.247
45	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.245
46	cvt_cvt-13-224-in1k_4_LucyV4	.242
47	resnet_50_v2	.242
48	cvt_cvt-21-224-in1k_4_LucyV4	.234
49	pnasnet_large-tf	.232
50	nasnet_large	.221
51	resnet-152_v2_pytorch	.220
52	resnet-152_v2-tf	.220
53	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.213
54	inception_v4	.212
55	inception_v3	.207
56	resnet_101_v2	.205
57	xception-keras	.204
58	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.202
59	convnext_tiny_sup	.198
60	deit_large_imagenet_full_seed-0	.198
61	resnet101	.191
62	resnet152_imagenet_full	.177
63	inception_v3-tf	.172
64	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.169
65	vgg_19	.163
66	resnet101_imagenet_full	.163
67	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.161
68	resnet50_random_l2_perturb	.161
69	nasnet_large-tf	.160
70	antialias-resnet152	.159
71	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.157
72	resnet_101_v1	.155
73	resnet50_imagenet_full	.155
74	deit_base_imagenet_full_seed-0	.153
75	inception_v4-tf	.153
76	resnet50-VITO-8deg-cc	.149
77	densenet-121-keras	.147
78	resnet-50_v2-tf	.145
79	AdvProp_efficientnet-b6	.143
80	resnet50_ewconplus_wzc	.143
81	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.142
82	densenet-121	.141
83	antialiased-rnext101_32x8d	.139
84	AdvProp_efficientnet-b4	.138
85	AT_efficientnet-b4	.138
86	resnet50_random_linf8_perturb	.136
87	resnet50	.135
88	efficientnet_b0	.132
89	resnet_152_v1	.131
90	resnet_50_v1_spiking	.127
91	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.127
92	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.125
93	efficientnet-b7	.125
94	efficientnet_b0_imagenet_full	.125
95	alexnet_fov4	.124
96	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.124
97	densenet-169-keras	.124
98	densenet-201	.121
99	grcnn_109	.120
100	Res2Net50_26w_4s	.119
101	densenet-169	.119
102	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.119
103	densenet-201-keras	.118
104	resnet50_byol	.118
105	cornet_s	.117
106	efficientnet_b2_imagenet_full	.117
107	AdvProp_efficientnet-b7	.116
108	AT_efficientnet-b2	.115
109	AdvProp_efficientnet-b2	.115
110	antialiased-r50	.115
111	BiT-S-R101x3	.114
112	resnet50-SIN_IN_IN	.114
113	resnet_SIN_IN_FT_IN	.114
114	resnet-34	.113
115	custom_model_cv_18_dagger_408	.112
116	resnet18_imagenet_full	.112
117	mobilenet_v2_1_0_192	.112
118	resnet34_imagenet_full	.111
119	convnext_small_imagenet_100_seed-0	.111
120	BiT-S-R152x4	.111
121	BiT-S-R50x3	.110
122	vgg_16	.109
123	resnet-152_v1-tf	.107
124	mobilenet_v2_1_0_224	.105
125	nasnet_mobile-tf	.104
126	mobilenet_v2_1_3_224	.104
127	resnet50_tutorial	.103
128	resnet_50_v1	.103
129	resnet-50-pytorch	.103
130	resnet50-sup	.103
131	deit_small_imagenet_full_seed-0	.102
132	BiT-S-R50x1	.102
133	resnet50-SIN_IN	.101
134	tv_efficientnet-b1	.101
135	grcnn	.099
136	mobilenet_v2_1.4_224-tf	.098
137	xception	.097
138	voneresnet-50-non_stochastic	.097
139	nasnet_mobile	.097
140	ViT_B_16_imagenet1k	.095
141	inception_v1	.095
142	efficientnet_b1_imagenet_full	.094
143	resnet50-moclr8deg	.093
144	ViT_L_32_imagenet1k	.092
145	ReAlnet06	.091
146	mobilenet_v2_0_5_224	.091
147	ReAlnet01	.090
148	mobilenet_v2_1.3_224-tf	.090
149	mobilenet_v1_1.0_224	.088
150	resnet50_moco_v2	.088
151	inception_v2-tf	.088
152	ReAlnet03	.088
153	resnet50-vitoimagevidnet8	.086
154	AlexNet_SIN_fov4	.086
155	resnet-101_v1-tf	.084
156	BiT-S-R152x2	.083
157	ReAlnet04	.082
158	ReAlnet02	.080
159	CORnet-S	.078
160	mobilenet_v2_1_0_160	.077
161	mobilenet_v2_0_75_224	.076
162	mobilenet_v1_1.0_192	.075
163	ReAlnet09	.075
164	inception_v1-tf	.073
165	vonegrcnn_52e_full	.073
166	mobilenet_v2_1.0_224-tf	.072
167	resnet-18	.072
168	mobilenet_v1_1.0_160	.070
169	ReAlnet07	.070
170	ReAlnet05	.070
171	ReAlnet08	.069
172	blt_vs	.068
173	vonegrcnn_52e_full	.068
174	vonegrcnn_62e_nobn	.068
175	AdvProp_efficientnet-b8	.068
176	resnet-50_v1-tf	.067
177	ViT_B_32_imagenet1k	.067
178	ReAlnet10	.067
179	regnet_y_400mf	.066
180	mobilenet_v2_0_5_192	.066
181	mobilenet_v2_1-4_224_pytorch	.064
182	mobilenet_v2_1_4_224	.064
183	vonegrcnn_62e_nobn	.064
184	vgg-16-keras	.063
185	resnet_50_v1_spiking_l4a	.063
186	vonegrcnn_47e	.063
187	mobilenet_v2_1.0_192-tf	.062
188	voneresnet-50	.062
189	vonegrcnn_47e	.061
190	mobilenet_v2_0_75_160	.060
191	mobilenet_v2_1.0_160-tf	.058
192	mobilenet_v1_0.75_160	.058
193	mobilenet_v2_0.75_160-tf	.057
194	BiT-S-R101x1	.057
195	resnet50_linf_4_robust	.056
196	mobilenet_v2_0.75_224-tf	.056
197	mobilenet_v2_1_0_128	.055
198	mobilenet_v2_0_75_192	.055
199	squeezenet1_0	.055
200	mobilenet_v1_0.5_224	.054
201	mobilenet_v2_0.35_160	.053
202	mobilenet_v1_0.75_224	.053
203	mobilenet_v2_0.5_192-tf	.053
204	squeezenet1_1	.052
205	mobilenet_v2_0.75_128	.052
206	resnet_50_v1_spiking_l4	.052
207	mobilenet_v2_0.5_224-tf	.051
208	voneresnet-50-robust	.051
209	mobilenet_v2_0.75_192-tf	.049
210	resnet50_imagenet_100_seed-0	.049
211	mobilenet_v2_0.5_160	.049
212	mobilenet_v1_0.75_192	.049
213	mobilenet_v2_0.5_128	.048
214	alexnet_reduced_aliasing_early_checkpoint	.048
215	alexnet	.047
216	mobilenet_v1_0.5_192	.047
217	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.047
218	shufflenet_v2_x1_0	.046
219	alexnet_random_l2_3_perturb	.046
220	vonealexnet_gaussian_noise_std4_fixed	.046
221	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.046
222	alexnet_random_linf8_perturb	.045
223	mobilenet_v2_0.35_192	.045
224	resnet50_l2_3_robust	.045
225	resnet50_eMMCR_Vanilla	.044
226	mobilenet_v1_0.75_128	.043
227	resnet50_robust_l2_eps1	.043
228	mobilenet_v2_1.0_96	.043
229	resnet50_linf_8_robust	.042
230	texture_shape_resnet50_trained_on_SIN	.042
231	mobilenet_v2_0.35_224	.042
232	mobilenet_v2_0.35_128	.042
233	mobilenet_v2_1.0_128-tf	.041
234	resnet50_simclr	.041
235	alexnet-baseline	.041
236	alexnet	.041
237	alexnet	.041
238	alexnet	.041
239	alexnet	.041
240	alexnet	.041
241	alexnet	.041
242	alexnet	.041
243	alexnet	.041
244	alexnet	.041
245	alexnet	.041
246	alexnet_ks_torevert	.041
247	vgg-19-keras	.040
248	alexnet_early_checkpoint	.039
249	mobilenet_v1_0.5_128	.039
250	AdvProp_efficientnet-b0	.039
251	alexnet_robust_correct	.039
252	alexnet_l2_3_robust	.038
253	mobilenet_v2_0.75_96	.038
254	mobilenet_v2_0.35_96	.038
255	alexnet_linf_8_robust	.036
256	mobilenet_v1_0.5_160	.035
257	0.5x_resnet-18_LC_w_sh_1_iter	.034
258	konkle_alexnetgn_ipcl_ref01_primary_model	.034
259	mobilenet_v1_1.0_128	.032
260	mobilenet_v2_0.5_96	.032
261	deit_small_imagenet_100_seed-0	.032
262	mobilenet_v1_0.25_192	.032
263	resnet50-vicreg	.032
264	mobilenet_v1_0.25_224	.031
265	AlexNet_SIN	.031
266	texture_shape_alexnet_trained_on_SIN	.031
267	resnet-50-robust	.030
268	imagenet_l2_3_0	.030
269	resnet50_robust_l2_eps3	.030
270	mobilenet_v1_0.25_160	.029
271	imagenet_l2_10_0	.028
272	AlexNet_SIN_fov12	.027
273	alexnet_fov12	.027
274	AlexNet_SIN_fov16	.026
275	resnet50-simclr-vissl	.026
276	deit_small_imagenet_10_seed-0	.025
277	convnext_small_imagenet_10_seed-0	.025
278	deit_small_imagenet_1_seed-0	.025
279	alexnet_training_seed_02	.024
280	alexnet_training_seed_04	.024
281	alexnet_training_seed_08	.024
282	alexnet_training_seed_10	.024
283	alexnet_training_seed_07	.024
284	alexnet_training_seed_09	.023
285	alexnet_fov16	.023
286	resnet_50_v1_spiking_l3	.023
287	resnet50-meshes-lt-100-original-pretrained	.023
288	resnet-50_untrained	.022
289	resnet50-SIN	.022
290	mobilenet_v1_0.25_128	.021
291	resnet50_imagenet_10_seed-0	.020
292	convnext_small_imagenet_1_seed-0	.020
293	resnet50-meshes-lt-100-original-scratch	.018
294	barlow-twins-resnet50	.014
295	resnet50-vicregl0p9	.014
296	artResNet18_1	.014
297	resnet50_FractalDB	.011
298	alexnet_training_seed_06	.009
299	alexnet_training_seed_03	.005
300	resnet50_imagenet_1_seed-0	.003
301	resnet50-vicregl0p75	.003
302	alexnet_training_seed_05	.002
303	alexnet_training_seed_01	.002
304	alexnet_training_seed_01_fov4	.002
305	alexnet_training_seed_01_fov12	.002
306	alexnet_training_seed_01_fov16	.002
307	resnet50-barlow	.001
308	resnet-18-LC_w_sh_100_iter_conv_init_m
309	ViT-B/32
310	BiT-M-R50x3
311	BiT-M-R50x1
312	BiT-M-R152x2
313	resnet-18-LC_w_sh_1_iter_conv_init_m
314	resnet-18-LC_w_sh_1_iter_m
315	resnet-18-LC_w_sh_10_iter_m
316	resnet-18_test_m
317	resnet-18-LC_w_sh_100_iter_m
318	resnet-18-LC_w_sh_10_iter_conv_init_m
319	FrankRobWobv0
320	0.5x_resnet-18_LC
321	CORnetZ_CIFAR10
322	resnet18-local_aggregation
323	resnet18-deepcluster
324	r3m_resnet18
325	CORnet-Z
326	vggface
327	my-model
328	resnet18-instance_recognition
329	0.5x_resnet-18_LC_w_sh_10_iter
330	r3m_resnet50
331	r3m_resnet34
332	resnet-18-LC_m
333	r3m_resnet50_nocrop
334	resnet18-supervised
335	resnet-18-LC_1st_conv_m
336	resnet-18-LC_conv_init_m
337	resnet18-contrastive_multiview
338	0.5x_resnet-18
339	resnet18-colorization
340	RN50
341	0.5x_resnet-18_LC_w_sh_100_iter
342	RN50
343	resnet18-relative_position
344	resnet18-autoencoder
345	resnet18-simclr
346	omnivore_swinT
347	ViT_L_32
348	ViT_B_32
349	resnet18-depth_prediction
350	ViT_B_16
351	r3m_resnet34_nocrop
352	resnet18-contrastive_predictive
353	resnet-18-LC_untrained
354	resnet-18_untrained
355	resnet-50x2_untrained
356	dcgan
357	unet_entire
358	omnivore_swinS
359	v1-pyr-nodown
360	CORnetZ_CIFAR10_bs32_20_04
361	pixels
362	dorinet_cornet_z
363	my-model
364	resnet50
365	openclip
366	resnet50-cifar
367	resnet50_pretrained_with_retinal_waves
368	resnet50_ImageNet
369	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
370	yudixie_resnet50_random_0_240908
371	yudixie_resnet50_imagenet1kpret_0_240908
372	yudixie_resnet50_imagenet1kpret_0_240312
373	yudixie_resnet50_object_class_0_240908
374	yudixie_resnet50_category_class_0_240908
375	ConvLSTM
376	yudixie_resnet50_distance_reg_0_240908
377	PredRNN
378	SimVP
379	TAU
380	briaai_rmbg_1_4
381	MIM
382	hmax
383	densenet_201_pytorch
384	pnasnet_large_pytorch
385	ReAlnet01_cornet
386	inception_v3_pytorch
387	yolos_tiny
388	ReAlnet03_cornet
389	yudixie_resnet18_imagenet1kpret_0_240719
390	ReAlnet02_cornet
391	compact_resnet50_V4
392	ReAlnet01_cornet_fov16
393	resnet18_ecoset_full
394	resnet34_ecoset_full
395	ReAlnet01_cornet_fov4
396	resnet50_ecoset_full
397	resnet101_ecoset_full
398	compact_resnet50_robust_V4
399	ReAlnet04_cornet
400	resnet152_ecoset_full
401	yudixie_resnet50_distance_translation_0_240908
402	yudixie_resnet18_distance_translation_0_240719
403	yudixie_resnet18_translation_rotation_0_240719
404	yudixie_resnet18_distance_rotation_0_240719
405	compact_resnet50_IT
406	ReAlnet01_cornet_fov12
407	yudixie_resnet18_category_class_0_240719
408	yudixie_resnet50_distance_rotation_0_240908
409	ReAlnet05_cornet
410	yudixie_resnet50_rotation_reg_0_240908
411	yudixie_resnet18_random_0_240719
412	compact_vgg19_V4
413	yudixie_resnet18_distance_translation_rotation_0_240719
414	yudixie_resnet18_object_class_0_240719
415	SeLaVi-AVE
416	yudixie_resnet18_translation_reg_0_240719
417	yudixie_resnet18_distance_reg_0_240719
418	ReAlnet06_cornet
419	yudixie_resnet50_translation_reg_0_240908
420	r3d_18
421	compact_vgg19_IT
422	ReAlnet01_ALL
423	mc3_18
424	ReAlnet07_cornet
425	yudixie_resnet50_translation_rotation_0_240908
426	s3d
427	ReAlnet08_cornet
428	mvit_v1_b
429	ReAlnet02_ALL
430	yudixie_resnet18_rotation_reg_0_240719
431	omnivore_swinB
432	VOneCORnet-S
433	compact_cornets_V4
434	resnet18_imagenet21kP
435	ReAlnet03_ALL
436	ReAlnet09_cornet
437	ReAlnet04_ALL
438	SeLaVi-Kinetics400
439	compact_cornets_IT
440	SeLaVi-VGG-Sound
441	ReAlnet05_ALL
442	r2plus1d_18
443	ReAlnet06_ALL
444	ReAlnet10_cornet
445	SeLaVi-Kinetics-Sound
446	mvit_v2_s
447	yudixie_resnet50_distance_translation_rotation_0_240908
448	ReAlnet07_ALL
449	compact_resnet50_robust_IT
450	ReAlnet08_ALL
451	yudixie_resnet18_cat_obj_class_all_latents_0_240719
452	resnet50_primary_visual_cortex
453	ReAlnet09_ALL
454	yudixie_resnet50_cat_obj_class_all_latents_0_240908
455	ReAlnet10_ALL

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.37.

Note that scores are relative to this ceiling.

Data: Geirhos2021sketch

Metric: error_consistency