Leaderboard Explore Compare Profile Community Blog FAQ Competition 2024 Report Issue

Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021colour-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	SWSL_resnext101_32x8d	.891
2	resnext101_32x8d_wsl	.885
3	convnext_femto_ols:d1_in1k	.866
4	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.815
5	pnasnet_large-tf	.812
6	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.811
7	cvt_cvt-13-224-in1k_4_LucyV4	.804
8	pnasnet_large	.800
9	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.799
10	vision_transformer_vit_large_patch16_224	.794
11	effnetb1_272x240	.792
12	effnetv2m_custom384	.788
13	nasnet_large-tf	.788
14	resnext101_32x48d_wsl	.787
15	nasnet_large	.769
16	CLIP-RN50	.769
17	SWSL_resnet50	.766
18	vit_large_patch14_clip_224:laion2b_ft_in1k	.762
19	resnext101_32x32d_wsl	.756
20	resnet50-VITO-8deg-cc	.755
21	CLIP_ViT-B_32	.748
22	convnext_tiny:in12k_ft_in1k	.746
23	CLIP_resnet50_float32	.739
24	CLIP_resnet50	.739
25	resnet_50_v2	.738
26	resnet_101_v2	.731
27	cvt_cvt-21-384-in1k_4_LucyV4	.727
28	resnet-152_v2-tf	.721
29	resnet-152_v2_pytorch	.716
30	ViT_L_16_imagenet1k	.715
31	cvt_cvt-21-224-in1k_4_LucyV4	.703
32	resnext101_32x16d_wsl	.696
33	resnet152_imagenet_full	.692
34	resnet_152_v2	.691
35	inception_v3	.689
36	inception_v4	.685
37	efficientnet_b0	.684
38	fixres_resnext101_32x48d_wsl	.675
39	resnet101	.675
40	cvt_cvt-13-384-in1k_4_LucyV4	.670
41	inception_v3-tf	.657
42	resnet101_imagenet_full	.653
43	focalnet_tiny_lrf_in1k	.647
44	ViT_B_16_imagenet1k	.640
45	tv_efficientnet-b1	.637
46	resnet50	.631
47	resnet_152_v1	.624
48	xception	.619
49	resnet50_random_l2_perturb	.617
50	inception_v4-tf	.591
51	resnet-50_v2-tf	.587
52	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.587
53	resnet50_imagenet_full	.585
54	antialiased-r50	.574
55	xception-keras	.561
56	vit_relpos_base_patch16_clsgap_224:sw_in1k	.560
57	resnet_101_v1	.551
58	AdvProp_efficientnet-b6	.546
59	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.543
60	vit_base_patch16_clip_224:openai_ft_in1k	.541
61	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.538
62	convnext_tiny_imagenet_full_seed-0	.534
63	antialiased-rnext101_32x8d	.528
64	resnet50_byol	.521
65	AdvProp_efficientnet-b7	.521
66	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.519
67	antialias-resnet152	.512
68	efficientnet-b7	.510
69	vgg_19	.505
70	resnet50_moco_v2	.504
71	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.499
72	Res2Net50_26w_4s	.491
73	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.491
74	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.489
75	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.488
76	densenet-201-keras	.479
77	resnet34_imagenet_full	.478
78	resnet-50-pytorch	.474
79	resnet50-sup	.474
80	resnet50_tutorial	.474
81	resnet_50_v1	.474
82	grcnn_109	.474
83	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.472
84	convnext_large:fb_in22k_ft_in1k	.470
85	AT_efficientnet-b2	.468
86	AdvProp_efficientnet-b2	.468
87	resnet50-SIN_IN_IN	.464
88	resnet_SIN_IN_FT_IN	.464
89	ViT_L_32_imagenet1k	.463
90	convnext_small_imagenet_full_seed-0	.462
91	resnet50-moclr8deg	.461
92	densenet-169	.456
93	deit_small_imagenet_full_seed-0	.454
94	AdvProp_efficientnet-b4	.450
95	AT_efficientnet-b4	.450
96	resnet_50_v1_spiking	.448
97	densenet-169-keras	.448
98	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.448
99	densenet-121-keras	.448
100	cornet_s	.445
101	resnet18_imagenet_full	.443
102	densenet_201_pytorch	.442
103	convnext_xlarge:fb_in22k_ft_in1k	.441
104	grcnn	.439
105	densenet-201	.438
106	resnet-101_v1-tf	.432
107	mobilenet_v2_1_3_224	.430
108	efficientnet_b2_imagenet_full	.429
109	vit_large_patch14_clip_224:openai_ft_in1k	.428
110	resnet50_random_linf8_perturb	.427
111	convnext_base:clip_laiona_augreg_ft_in1k_384	.422
112	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.419
113	deit_large_imagenet_full_seed-0	.411
114	convnext_small_imagenet_100_seed-0	.406
115	deit_base_imagenet_full_seed-0	.406
116	densenet-121	.404
117	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.403
118	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.403
119	cv_18_dagger_408_pretrained	.400
120	resnet-50_v1-tf	.395
121	convnext_large_imagenet_full_seed-0	.395
122	BiT-S-R50x3	.391
123	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.390
124	vit_large_patch14_clip_336:laion2b_ft_in1k	.387
125	ReAlnet03	.377
126	resnet-152_v1-tf	.373
127	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.370
128	ReAlnet01	.370
129	convnext_base_imagenet_full_seed-0	.365
130	mobilenet_v2_1.4_224-tf	.363
131	voneresnet-50-non_stochastic	.361
132	BiT-S-R152x2	.356
133	custom_model_cv_18_dagger_408	.346
134	resnet50-SIN_IN	.344
135	inception_v2-tf	.344
136	mobilenet_v2_1.3_224-tf	.343
137	ReAlnet04	.342
138	vgg_16	.341
139	ReAlnet08	.328
140	ReAlnet09	.325
141	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.324
142	ReAlnet05	.322
143	efficientnet_b0_imagenet_full	.320
144	ViT_B_32_imagenet1k	.320
145	ReAlnet07	.316
146	CORnet-S	.314
147	ReAlnet06	.314
148	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.311
149	nasnet_mobile-tf	.309
150	inception_v1	.300
151	ReAlnet02	.299
152	vonegrcnn_62e_nobn	.298
153	vonegrcnn_62e_nobn	.293
154	efficientnet_b1_imagenet_full	.290
155	ReAlnet10	.290
156	mobilenet_v2_1-4_224_pytorch	.288
157	mobilenet_v2_1_4_224	.288
158	inception_v1-tf	.288
159	nasnet_mobile	.286
160	vonegrcnn_47e	.284
161	BiT-S-R50x1	.269
162	resnet50_linf_4_robust	.268
163	AdvProp_efficientnet-b8	.263
164	BiT-S-R101x3	.263
165	BiT-S-R152x4	.261
166	resnet-18	.260
167	vonegrcnn_47e	.260
168	convnext_tiny_sup	.254
169	mobilenet_v2_0_5_224	.253
170	resnet-34	.252
171	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.248
172	regnet_y_400mf	.248
173	mobilenet_v2_1_0_192	.246
174	vonegrcnn_52e_full	.239
175	vonegrcnn_52e_full	.236
176	mobilenet_v2_0_75_224	.231
177	mobilenet_v2_1_0_160	.228
178	voneresnet-50	.218
179	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.216
180	mobilenet_v1_1.0_224	.215
181	blt_vs	.214
182	mobilenet_v2_1_0_224	.214
183	vgg-16-keras	.211
184	mobilenet_v1_0.75_224	.211
185	vgg-19-keras	.188
186	BiT-S-R101x1	.182
187	texture_shape_resnet50_trained_on_SIN	.182
188	voneresnet-50-robust	.180
189	resnet50_eMMCR_Vanilla	.179
190	mobilenet_v2_1.0_160-tf	.177
191	mobilenet_v1_1.0_160	.173
192	resnet50-vitoimagevidnet8	.170
193	mobilenet_v2_1.0_224-tf	.168
194	mobilenet_v2_0_5_192	.165
195	alexnet	.163
196	alexnet_random_linf8_perturb	.162
197	mobilenet_v2_1.0_192-tf	.161
198	mobilenet_v2_0.75_224-tf	.159
199	alexnet_random_l2_3_perturb	.152
200	resnet50_robust_l2_eps1	.152
201	mobilenet_v2_0.5_224-tf	.150
202	mobilenet_v2_0_75_192	.150
203	resnet50_l2_3_robust	.146
204	resnet_50_v1_spiking_l4a	.143
205	resnet50_simclr	.143
206	alexnet_reduced_aliasing_early_checkpoint	.137
207	mobilenet_v2_0_75_160	.134
208	alexnet-baseline	.134
209	alexnet	.134
210	alexnet	.134
211	alexnet	.134
212	alexnet	.134
213	alexnet	.134
214	alexnet	.134
215	alexnet	.134
216	alexnet	.134
217	alexnet	.134
218	alexnet_ks_torevert	.134
219	konkle_alexnetgn_ipcl_ref01_primary_model	.134
220	mobilenet_v2_0.5_192-tf	.131
221	shufflenet_v2_x1_0	.130
222	mobilenet_v1_0.5_224	.123
223	mobilenet_v2_1_0_128	.122
224	resnet50_imagenet_100_seed-0	.120
225	alexnet_early_checkpoint	.119
226	AdvProp_efficientnet-b0	.115
227	mobilenet_v1_0.75_192	.113
228	resnet50-SIN	.111
229	mobilenet_v2_0.75_160-tf	.108
230	vonealexnet_gaussian_noise_std4_fixed	.104
231	resnet-50-robust	.104
232	imagenet_l2_3_0	.104
233	resnet50_robust_l2_eps3	.104
234	resnet50_linf_8_robust	.104
235	mobilenet_v1_1.0_192	.096
236	mobilenet_v1_0.5_192	.090
237	0.5x_resnet-18_LC_w_sh_1_iter	.084
238	mobilenet_v2_0.75_192-tf	.072
239	mobilenet_v2_0.75_96	.072
240	texture_shape_alexnet_trained_on_SIN	.071
241	mobilenet_v1_0.75_160	.068
242	mobilenet_v1_1.0_128	.068
243	squeezenet1_0	.067
244	AlexNet_SIN	.065
245	mobilenet_v2_0.75_128	.062
246	mobilenet_v2_1.0_128-tf	.061
247	alexnet_linf_8_robust	.060
248	alexnet_l2_3_robust	.060
249	mobilenet_v2_0.35_160	.058
250	alexnet_training_seed_07	.050
251	alexnet_training_seed_02	.049
252	alexnet_training_seed_04	.049
253	alexnet_training_seed_08	.049
254	alexnet_training_seed_10	.049
255	mobilenet_v2_0.35_192	.048
256	squeezenet1_1	.047
257	mobilenet_v1_0.25_192	.045
258	mobilenet_v1_0.25_224	.044
259	alexnet_robust_correct	.043
260	resnet50-vicreg	.043
261	imagenet_l2_10_0	.041
262	mobilenet_v1_0.75_128	.039
263	mobilenet_v2_0.35_128	.038
264	mobilenet_v2_0.5_128	.037
265	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.037
266	mobilenet_v1_0.5_160	.036
267	mobilenet_v2_0.35_96	.036
268	mobilenet_v2_0.35_224	.035
269	deit_small_imagenet_100_seed-0	.035
270	resnet50-vicregl0p9	.031
271	mobilenet_v1_0.25_128	.030
272	mobilenet_v2_1.0_96	.030
273	convnext_small_imagenet_10_seed-0	.030
274	deit_small_imagenet_10_seed-0	.027
275	resnet50_imagenet_10_seed-0	.027
276	deit_small_imagenet_1_seed-0	.027
277	alexnet_training_seed_03	.025
278	mobilenet_v2_0.5_160	.025
279	mobilenet_v2_0.5_96	.025
280	resnet50-meshes-lt-100-original-pretrained	.022
281	alexnet_training_seed_06	.022
282	resnet50-barlow	.020
283	mobilenet_v1_0.5_128	.020
284	alexnet_training_seed_05	.020
285	alexnet_training_seed_01	.020
286	resnet_50_v1_spiking_l4	.019
287	artResNet18_1	.015
288	my-model	.014
289	resnet50-vicregl0p75	.014
290	resnet_50_v1_spiking_l3	.012
291	barlow-twins-resnet50	.009
292	resnet-50_untrained	.009
293	resnet50-meshes-lt-100-original-scratch	.009
294	convnext_small_imagenet_1_seed-0	.009
295	alexnet_training_seed_09	.007
296	mobilenet_v1_0.25_160	.006
297	resnet50_FractalDB	.004
298	mobilevit_small	.004
299	resnet50_imagenet_1_seed-0	.003
300	resnet50-simclr-vissl	.002
301	resnet-18-LC_w_sh_100_iter_conv_init_m
302	ViT-B/32
303	BiT-M-R50x3
304	BiT-M-R152x4
305	BiT-M-R50x1
306	BiT-M-R152x2
307	resnet-18-LC_w_sh_1_iter_conv_init_m
308	resnet-18-LC_w_sh_1_iter_m
309	resnet-18-LC_w_sh_10_iter_m
310	resnet-18-LC_w_sh_100_iter_m
311	resnet-18-LC_w_sh_10_iter_conv_init_m
312	FrankRobWobv0
313	resnet18-local_aggregation
314	0.5x_resnet-18_LC
315	CORnetZ_CIFAR10
316	resnet18-colorization
317	resnet18-deepcluster
318	r3m_resnet18
319	resnet18-instance_recognition
320	CORnet-Z
321	my-model
322	vggface
323	0.5x_resnet-18_LC_w_sh_10_iter
324	r3m_resnet50
325	resnet18-relative_position
326	r3m_resnet34
327	resnet-18-LC_m
328	r3m_resnet50_nocrop
329	resnet18-supervised
330	resnet-18-LC_1st_conv_m
331	resnet-18-LC_conv_init_m
332	resnet18-contrastive_multiview
333	0.5x_resnet-18
334	RN50
335	0.5x_resnet-18_LC_w_sh_100_iter
336	RN50
337	resnet18-autoencoder
338	resnet18-simclr
339	ViT_L_32
340	ViT_B_32
341	resnet18-depth_prediction
342	ViT_B_16
343	r3m_resnet34_nocrop
344	resnet18-contrastive_predictive
345	omnivore_swinT
346	resnet-18-LC_untrained
347	resnet-18_untrained
348	resnet-50x2_untrained
349	dcgan
350	v1-pyr-nodown
351	unet_entire
352	CORnetZ_CIFAR10_bs32_20_04
353	pixels
354	dorinet_cornet_z
355	resnet50
356	openclip
357	resnet50-cifar
358	resnet50_pretrained_with_retinal_waves
359	resnet50_ImageNet
360	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
361	yudixie_resnet50_random_0_240908
362	yudixie_resnet50_imagenet1kpret_0_240908
363	yudixie_resnet50_cat_obj_class_all_latents_0_240908
364	yudixie_resnet50_object_class_0_240908
365	yudixie_resnet50_category_class_0_240908
366	resnet18_imagenet21kP
367	ConvLSTM
368	PredRNN
369	yudixie_resnet50_distance_reg_0_240908
370	SimVP
371	TAU
372	MIM
373	hmax
374	ReAlnet01_cornet
375	pnasnet_large_pytorch
376	inception_v3_pytorch
377	briaai_rmbg_1_4
378	yudixie_resnet50_imagenet1kpret_0_240312
379	yudixie_resnet18_translation_reg_0_240719
380	yudixie_resnet18_rotation_reg_0_240719
381	yudixie_resnet18_imagenet1kpret_0_240719
382	ReAlnet02_cornet
383	resnet18_ecoset_full
384	resnet50_ecoset_full
385	resnet101_ecoset_full
386	yudixie_resnet50_distance_rotation_0_240908
387	yudixie_resnet50_rotation_reg_0_240908
388	resnet152_ecoset_full
389	yudixie_resnet18_distance_translation_0_240719
390	ReAlnet04_cornet
391	yudixie_resnet50_distance_translation_0_240908
392	yudixie_resnet18_translation_rotation_0_240719
393	ReAlnet05_cornet
394	yudixie_resnet18_random_0_240719
395	yudixie_resnet18_cat_obj_class_all_latents_0_240719
396	resnet-18_test_m
397	SeLaVi-AVE
398	yudixie_resnet18_distance_reg_0_240719
399	r3d_18
400	ReAlnet06_cornet
401	yudixie_resnet50_translation_reg_0_240908
402	ReAlnet01_ALL
403	mc3_18
404	ReAlnet07_cornet
405	yudixie_resnet50_translation_rotation_0_240908
406	s3d
407	mvit_v1_b
408	ReAlnet08_cornet
409	ReAlnet02_ALL
410	ReAlnet03_ALL
411	VOneCORnet-S
412	ReAlnet04_ALL
413	omnivore_swinB
414	SeLaVi-Kinetics400
415	ReAlnet09_cornet
416	SeLaVi-VGG-Sound
417	ReAlnet05_ALL
418	SeLaVi-Kinetics-Sound
419	omnivore_swinS
420	ReAlnet06_ALL
421	mvit_v2_s
422	r2plus1d_18
423	ReAlnet07_ALL
424	resnet34_ecoset_full
425	yudixie_resnet18_object_class_0_240719
426	yudixie_resnet18_category_class_0_240719
427	ReAlnet08_ALL
428	yudixie_resnet50_distance_translation_rotation_0_240908
429	yudixie_resnet18_distance_translation_rotation_0_240719
430	yudixie_resnet18_distance_rotation_0_240719
431	ReAlnet03_cornet
432	ReAlnet09_ALL
433	ReAlnet10_cornet
434	resnet50_primary_visual_cortex
435	ReAlnet10_ALL
436	yolos_tiny

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.42.

Note that scores are relative to this ceiling.

Data: Geirhos2021colour

Metric: error_consistency