Leaderboard Explore Compare Profile Community Blog FAQ Competition 2024 Report Issue

Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021colour-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	SWSL_resnext101_32x8d	.891
2	resnext101_32x8d_wsl	.885
3	convnext_femto_ols:d1_in1k	.866
4	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.815
5	pnasnet_large-tf	.812
6	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.811
7	cvt_cvt-13-224-in1k_4_LucyV4	.804
8	pnasnet_large	.800
9	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.799
10	vision_transformer_vit_large_patch16_224	.794
11	effnetb1_272x240	.792
12	effnetv2m_custom384	.788
13	nasnet_large-tf	.788
14	resnext101_32x48d_wsl	.787
15	nasnet_large	.769
16	CLIP-RN50	.769
17	SWSL_resnet50	.766
18	vit_large_patch14_clip_224:laion2b_ft_in1k	.762
19	resnext101_32x32d_wsl	.756
20	resnet50-VITO-8deg-cc	.755
21	CLIP_ViT-B_32	.748
22	convnext_tiny:in12k_ft_in1k	.746
23	CLIP_resnet50_float32	.739
24	CLIP_resnet50	.739
25	resnet_50_v2	.738
26	resnet_101_v2	.731
27	cvt_cvt-21-384-in1k_4_LucyV4	.727
28	resnet-152_v2-tf	.721
29	resnet-152_v2_pytorch	.716
30	ViT_L_16_imagenet1k	.715
31	cvt_cvt-21-224-in1k_4_LucyV4	.703
32	resnext101_32x16d_wsl	.696
33	resnet152_imagenet_full	.692
34	resnet_152_v2	.691
35	inception_v3	.689
36	inception_v4	.685
37	efficientnet_b0	.684
38	fixres_resnext101_32x48d_wsl	.675
39	resnet101	.675
40	cvt_cvt-13-384-in1k_4_LucyV4	.670
41	inception_v3-tf	.657
42	resnet101_imagenet_full	.653
43	focalnet_tiny_lrf_in1k	.647
44	ViT_B_16_imagenet1k	.640
45	tv_efficientnet-b1	.637
46	resnet50	.631
47	resnet_152_v1	.624
48	xception	.619
49	resnet50_random_l2_perturb	.617
50	inception_v4-tf	.591
51	resnet-50_v2-tf	.587
52	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.587
53	resnet50_imagenet_full	.585
54	antialiased-r50	.574
55	xception-keras	.561
56	vit_relpos_base_patch16_clsgap_224:sw_in1k	.560
57	resnet_101_v1	.551
58	AdvProp_efficientnet-b6	.546
59	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.543
60	vit_base_patch16_clip_224:openai_ft_in1k	.541
61	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.538
62	convnext_tiny_imagenet_full_seed-0	.534
63	antialiased-rnext101_32x8d	.528
64	resnet50_byol	.521
65	AdvProp_efficientnet-b7	.521
66	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.519
67	antialias-resnet152	.512
68	efficientnet-b7	.510
69	alexnet_fov4	.505
70	vgg_19	.505
71	resnet50_moco_v2	.504
72	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.499
73	Res2Net50_26w_4s	.491
74	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.491
75	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.489
76	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.488
77	densenet-201-keras	.479
78	resnet34_imagenet_full	.478
79	resnet-50-pytorch	.474
80	resnet50-sup	.474
81	resnet50_tutorial	.474
82	resnet_50_v1	.474
83	grcnn_109	.474
84	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.472
85	convnext_large:fb_in22k_ft_in1k	.470
86	AT_efficientnet-b2	.468
87	AdvProp_efficientnet-b2	.468
88	resnet50_ewconplus_wzc	.466
89	resnet50-SIN_IN_IN	.464
90	resnet_SIN_IN_FT_IN	.464
91	ViT_L_32_imagenet1k	.463
92	convnext_small_imagenet_full_seed-0	.462
93	resnet50-moclr8deg	.461
94	densenet-169	.456
95	deit_small_imagenet_full_seed-0	.454
96	AdvProp_efficientnet-b4	.450
97	AT_efficientnet-b4	.450
98	resnet_50_v1_spiking	.448
99	densenet-169-keras	.448
100	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.448
101	densenet-121-keras	.448
102	cornet_s	.445
103	resnet18_imagenet_full	.443
104	densenet_201_pytorch	.442
105	convnext_xlarge:fb_in22k_ft_in1k	.441
106	grcnn	.439
107	densenet-201	.438
108	resnet-101_v1-tf	.432
109	mobilenet_v2_1_3_224	.430
110	efficientnet_b2_imagenet_full	.429
111	vit_large_patch14_clip_224:openai_ft_in1k	.428
112	resnet50_random_linf8_perturb	.427
113	convnext_base:clip_laiona_augreg_ft_in1k_384	.422
114	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.419
115	deit_large_imagenet_full_seed-0	.411
116	convnext_small_imagenet_100_seed-0	.406
117	deit_base_imagenet_full_seed-0	.406
118	densenet-121	.404
119	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.403
120	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.403
121	cv_18_dagger_408_pretrained	.400
122	resnet-50_v1-tf	.395
123	convnext_large_imagenet_full_seed-0	.395
124	BiT-S-R50x3	.391
125	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.390
126	vit_large_patch14_clip_336:laion2b_ft_in1k	.387
127	ReAlnet03	.377
128	resnet-152_v1-tf	.373
129	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.370
130	ReAlnet01	.370
131	convnext_base_imagenet_full_seed-0	.365
132	mobilenet_v2_1.4_224-tf	.363
133	voneresnet-50-non_stochastic	.361
134	BiT-S-R152x2	.356
135	custom_model_cv_18_dagger_408	.346
136	resnet50-SIN_IN	.344
137	inception_v2-tf	.344
138	mobilenet_v2_1.3_224-tf	.343
139	ReAlnet04	.342
140	vgg_16	.341
141	ReAlnet08	.328
142	ReAlnet09	.325
143	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.324
144	ReAlnet05	.322
145	efficientnet_b0_imagenet_full	.320
146	ViT_B_32_imagenet1k	.320
147	ReAlnet07	.316
148	CORnet-S	.314
149	ReAlnet06	.314
150	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.311
151	nasnet_mobile-tf	.309
152	inception_v1	.300
153	ReAlnet02	.299
154	vonegrcnn_62e_nobn	.298
155	vonegrcnn_62e_nobn	.293
156	efficientnet_b1_imagenet_full	.290
157	ReAlnet10	.290
158	mobilenet_v2_1-4_224_pytorch	.288
159	mobilenet_v2_1_4_224	.288
160	inception_v1-tf	.288
161	nasnet_mobile	.286
162	vonegrcnn_47e	.284
163	BiT-S-R50x1	.269
164	resnet50_linf_4_robust	.268
165	AdvProp_efficientnet-b8	.263
166	BiT-S-R101x3	.263
167	BiT-S-R152x4	.261
168	resnet-18	.260
169	vonegrcnn_47e	.260
170	AlexNet_SIN_fov4	.260
171	convnext_tiny_sup	.254
172	mobilenet_v2_0_5_224	.253
173	resnet-34	.252
174	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.248
175	regnet_y_400mf	.248
176	mobilenet_v2_1_0_192	.246
177	vonegrcnn_52e_full	.239
178	vonegrcnn_52e_full	.236
179	mobilenet_v2_0_75_224	.231
180	mobilenet_v2_1_0_160	.228
181	voneresnet-50	.218
182	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.216
183	mobilenet_v1_1.0_224	.215
184	blt_vs	.214
185	mobilenet_v2_1_0_224	.214
186	vgg-16-keras	.211
187	mobilenet_v1_0.75_224	.211
188	vgg-19-keras	.188
189	BiT-S-R101x1	.182
190	texture_shape_resnet50_trained_on_SIN	.182
191	voneresnet-50-robust	.180
192	resnet50_eMMCR_Vanilla	.179
193	mobilenet_v2_1.0_160-tf	.177
194	mobilenet_v1_1.0_160	.173
195	resnet50-vitoimagevidnet8	.170
196	mobilenet_v2_1.0_224-tf	.168
197	mobilenet_v2_0_5_192	.165
198	alexnet	.163
199	alexnet_random_linf8_perturb	.162
200	mobilenet_v2_1.0_192-tf	.161
201	mobilenet_v2_0.75_224-tf	.159
202	alexnet_random_l2_3_perturb	.152
203	resnet50_robust_l2_eps1	.152
204	mobilenet_v2_0.5_224-tf	.150
205	mobilenet_v2_0_75_192	.150
206	resnet50_l2_3_robust	.146
207	resnet_50_v1_spiking_l4a	.143
208	resnet50_simclr	.143
209	alexnet_reduced_aliasing_early_checkpoint	.137
210	mobilenet_v2_0_75_160	.134
211	alexnet-baseline	.134
212	alexnet	.134
213	alexnet	.134
214	alexnet	.134
215	alexnet	.134
216	alexnet	.134
217	alexnet	.134
218	alexnet	.134
219	alexnet	.134
220	alexnet	.134
221	alexnet	.134
222	alexnet_ks_torevert	.134
223	konkle_alexnetgn_ipcl_ref01_primary_model	.134
224	mobilenet_v2_0.5_192-tf	.131
225	shufflenet_v2_x1_0	.130
226	mobilenet_v1_0.5_224	.123
227	mobilenet_v2_1_0_128	.122
228	resnet50_imagenet_100_seed-0	.120
229	alexnet_early_checkpoint	.119
230	AdvProp_efficientnet-b0	.115
231	mobilenet_v1_0.75_192	.113
232	resnet50-SIN	.111
233	mobilenet_v2_0.75_160-tf	.108
234	vonealexnet_gaussian_noise_std4_fixed	.104
235	resnet-50-robust	.104
236	imagenet_l2_3_0	.104
237	resnet50_robust_l2_eps3	.104
238	resnet50_linf_8_robust	.104
239	mobilenet_v1_1.0_192	.096
240	mobilenet_v1_0.5_192	.090
241	0.5x_resnet-18_LC_w_sh_1_iter	.084
242	mobilenet_v2_0.75_192-tf	.072
243	mobilenet_v2_0.75_96	.072
244	texture_shape_alexnet_trained_on_SIN	.071
245	mobilenet_v1_0.75_160	.068
246	mobilenet_v1_1.0_128	.068
247	squeezenet1_0	.067
248	AlexNet_SIN	.065
249	mobilenet_v2_0.75_128	.062
250	mobilenet_v2_1.0_128-tf	.061
251	alexnet_linf_8_robust	.060
252	alexnet_l2_3_robust	.060
253	mobilenet_v2_0.35_160	.058
254	alexnet_training_seed_07	.050
255	alexnet_training_seed_02	.049
256	alexnet_training_seed_04	.049
257	alexnet_training_seed_08	.049
258	alexnet_training_seed_10	.049
259	mobilenet_v2_0.35_192	.048
260	squeezenet1_1	.047
261	mobilenet_v1_0.25_192	.045
262	mobilenet_v1_0.25_224	.044
263	alexnet_robust_correct	.043
264	resnet50-vicreg	.043
265	imagenet_l2_10_0	.041
266	mobilenet_v1_0.75_128	.039
267	mobilenet_v2_0.35_128	.038
268	alexnet_fov12	.038
269	mobilenet_v2_0.5_128	.037
270	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.037
271	mobilenet_v1_0.5_160	.036
272	mobilenet_v2_0.35_96	.036
273	mobilenet_v2_0.35_224	.035
274	deit_small_imagenet_100_seed-0	.035
275	resnet50-vicregl0p9	.031
276	mobilenet_v1_0.25_128	.030
277	mobilenet_v2_1.0_96	.030
278	convnext_small_imagenet_10_seed-0	.030
279	deit_small_imagenet_10_seed-0	.027
280	resnet50_imagenet_10_seed-0	.027
281	deit_small_imagenet_1_seed-0	.027
282	alexnet_fov16	.026
283	alexnet_training_seed_03	.025
284	mobilenet_v2_0.5_160	.025
285	mobilenet_v2_0.5_96	.025
286	resnet50-meshes-lt-100-original-pretrained	.022
287	alexnet_training_seed_06	.022
288	AlexNet_SIN_fov12	.020
289	resnet50-barlow	.020
290	mobilenet_v1_0.5_128	.020
291	alexnet_training_seed_05	.020
292	alexnet_training_seed_01	.020
293	alexnet_training_seed_01_fov4	.020
294	alexnet_training_seed_01_fov12	.020
295	alexnet_training_seed_01_fov16	.020
296	resnet_50_v1_spiking_l4	.019
297	artResNet18_1	.015
298	my-model	.014
299	resnet50-vicregl0p75	.014
300	resnet_50_v1_spiking_l3	.012
301	AlexNet_SIN_fov16	.011
302	barlow-twins-resnet50	.009
303	resnet-50_untrained	.009
304	resnet50-meshes-lt-100-original-scratch	.009
305	convnext_small_imagenet_1_seed-0	.009
306	alexnet_training_seed_09	.007
307	mobilenet_v1_0.25_160	.006
308	resnet50_FractalDB	.004
309	mobilevit_small	.004
310	resnet50_imagenet_1_seed-0	.003
311	resnet50-simclr-vissl	.002
312	resnet-18-LC_w_sh_100_iter_conv_init_m
313	ViT-B/32
314	BiT-M-R50x3
315	BiT-M-R152x4
316	BiT-M-R50x1
317	BiT-M-R152x2
318	resnet-18-LC_w_sh_1_iter_conv_init_m
319	resnet-18-LC_w_sh_1_iter_m
320	resnet-18-LC_w_sh_10_iter_m
321	resnet-18-LC_w_sh_100_iter_m
322	resnet-18-LC_w_sh_10_iter_conv_init_m
323	FrankRobWobv0
324	resnet18-local_aggregation
325	0.5x_resnet-18_LC
326	CORnetZ_CIFAR10
327	resnet18-colorization
328	resnet18-deepcluster
329	r3m_resnet18
330	resnet18-instance_recognition
331	CORnet-Z
332	my-model
333	vggface
334	0.5x_resnet-18_LC_w_sh_10_iter
335	r3m_resnet50
336	resnet18-relative_position
337	r3m_resnet34
338	resnet-18-LC_m
339	r3m_resnet50_nocrop
340	resnet18-supervised
341	resnet-18-LC_1st_conv_m
342	resnet-18-LC_conv_init_m
343	resnet18-contrastive_multiview
344	0.5x_resnet-18
345	RN50
346	0.5x_resnet-18_LC_w_sh_100_iter
347	RN50
348	resnet18-autoencoder
349	resnet18-simclr
350	ViT_L_32
351	ViT_B_32
352	resnet18-depth_prediction
353	ViT_B_16
354	r3m_resnet34_nocrop
355	resnet18-contrastive_predictive
356	omnivore_swinT
357	resnet-18-LC_untrained
358	resnet-18_untrained
359	resnet-50x2_untrained
360	dcgan
361	v1-pyr-nodown
362	unet_entire
363	CORnetZ_CIFAR10_bs32_20_04
364	pixels
365	dorinet_cornet_z
366	resnet50
367	openclip
368	resnet50-cifar
369	resnet50_pretrained_with_retinal_waves
370	resnet50_ImageNet
371	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
372	yudixie_resnet50_random_0_240908
373	yudixie_resnet50_imagenet1kpret_0_240908
374	yudixie_resnet50_cat_obj_class_all_latents_0_240908
375	yudixie_resnet50_object_class_0_240908
376	yudixie_resnet50_category_class_0_240908
377	resnet18_imagenet21kP
378	ConvLSTM
379	PredRNN
380	yudixie_resnet50_distance_reg_0_240908
381	SimVP
382	TAU
383	MIM
384	hmax
385	ReAlnet01_cornet
386	pnasnet_large_pytorch
387	inception_v3_pytorch
388	briaai_rmbg_1_4
389	yudixie_resnet50_imagenet1kpret_0_240312
390	yudixie_resnet18_translation_reg_0_240719
391	yudixie_resnet18_rotation_reg_0_240719
392	compact_resnet50_V4
393	yudixie_resnet18_imagenet1kpret_0_240719
394	ReAlnet02_cornet
395	resnet18_ecoset_full
396	compact_resnet50_IT
397	ReAlnet01_cornet_fov4
398	resnet50_ecoset_full
399	resnet101_ecoset_full
400	yudixie_resnet50_distance_rotation_0_240908
401	yudixie_resnet50_rotation_reg_0_240908
402	resnet152_ecoset_full
403	yudixie_resnet18_distance_translation_0_240719
404	ReAlnet04_cornet
405	yudixie_resnet50_distance_translation_0_240908
406	ReAlnet01_cornet_fov12
407	yudixie_resnet18_translation_rotation_0_240719
408	compact_resnet50_robust_V4
409	compact_vgg19_V4
410	ReAlnet05_cornet
411	yudixie_resnet18_random_0_240719
412	yudixie_resnet18_cat_obj_class_all_latents_0_240719
413	resnet-18_test_m
414	SeLaVi-AVE
415	yudixie_resnet18_distance_reg_0_240719
416	r3d_18
417	ReAlnet06_cornet
418	yudixie_resnet50_translation_reg_0_240908
419	compact_vgg19_IT
420	ReAlnet01_ALL
421	mc3_18
422	ReAlnet07_cornet
423	yudixie_resnet50_translation_rotation_0_240908
424	s3d
425	ReAlnet01_cornet_fov16
426	mvit_v1_b
427	ReAlnet08_cornet
428	compact_cornets_V4
429	ReAlnet02_ALL
430	ReAlnet03_ALL
431	VOneCORnet-S
432	ReAlnet04_ALL
433	omnivore_swinB
434	SeLaVi-Kinetics400
435	ReAlnet09_cornet
436	SeLaVi-VGG-Sound
437	compact_cornets_IT
438	ReAlnet05_ALL
439	SeLaVi-Kinetics-Sound
440	omnivore_swinS
441	ReAlnet06_ALL
442	mvit_v2_s
443	compact_resnet50_robust_IT
444	r2plus1d_18
445	ReAlnet07_ALL
446	resnet34_ecoset_full
447	yudixie_resnet18_object_class_0_240719
448	yudixie_resnet18_category_class_0_240719
449	ReAlnet08_ALL
450	yudixie_resnet50_distance_translation_rotation_0_240908
451	yudixie_resnet18_distance_translation_rotation_0_240719
452	yudixie_resnet18_distance_rotation_0_240719
453	ReAlnet03_cornet
454	ReAlnet09_ALL
455	ReAlnet10_cornet
456	resnet50_primary_visual_cortex
457	ReAlnet10_ALL
458	yolos_tiny

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.42.

Note that scores are relative to this ceiling.

Data: Geirhos2021colour

Metric: error_consistency