Leaderboard Explore Compare Profile Community Blog FAQ Competition 2024 Report Issue

Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021colour-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	SWSL_resnext101_32x8d	.891
2	resnext101_32x8d_wsl	.885
3	convnext_femto_ols:d1_in1k	.866
4	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.815
5	pnasnet_large-tf	.812
6	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.811
7	cvt_cvt-13-224-in1k_4_LucyV4	.804
8	pnasnet_large	.800
9	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.799
10	vision_transformer_vit_large_patch16_224	.794
11	effnetb1_272x240	.792
12	effnetv2m_custom384	.788
13	nasnet_large-tf	.788
14	resnext101_32x48d_wsl	.787
15	nasnet_large	.769
16	CLIP-RN50	.769
17	SWSL_resnet50	.766
18	vit_large_patch14_clip_224:laion2b_ft_in1k	.762
19	resnext101_32x32d_wsl	.756
20	resnet50-VITO-8deg-cc	.755
21	CLIP_ViT-B_32	.748
22	convnext_tiny:in12k_ft_in1k	.746
23	CLIP_resnet50_float32	.739
24	CLIP_resnet50	.739
25	resnet_50_v2	.738
26	resnet_101_v2	.731
27	cvt_cvt-21-384-in1k_4_LucyV4	.727
28	resnet-152_v2-tf	.721
29	resnet-152_v2_pytorch	.716
30	ViT_L_16_imagenet1k	.715
31	cvt_cvt-21-224-in1k_4_LucyV4	.703
32	resnext101_32x16d_wsl	.696
33	resnet152_imagenet_full	.692
34	resnet_152_v2	.691
35	inception_v3	.689
36	inception_v4	.685
37	efficientnet_b0	.684
38	fixres_resnext101_32x48d_wsl	.675
39	resnet101	.675
40	cvt_cvt-13-384-in1k_4_LucyV4	.670
41	inception_v3-tf	.657
42	resnet101_imagenet_full	.653
43	focalnet_tiny_lrf_in1k	.647
44	ViT_B_16_imagenet1k	.640
45	tv_efficientnet-b1	.637
46	resnet50	.631
47	resnet_152_v1	.624
48	xception	.619
49	resnet50_random_l2_perturb	.617
50	inception_v4-tf	.591
51	resnet-50_v2-tf	.587
52	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.587
53	resnet50_imagenet_full	.585
54	antialiased-r50	.574
55	xception-keras	.561
56	vit_relpos_base_patch16_clsgap_224:sw_in1k	.560
57	resnet_101_v1	.551
58	AdvProp_efficientnet-b6	.546
59	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.543
60	vit_base_patch16_clip_224:openai_ft_in1k	.541
61	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.538
62	convnext_tiny_imagenet_full_seed-0	.534
63	antialiased-rnext101_32x8d	.528
64	resnet50_byol	.521
65	AdvProp_efficientnet-b7	.521
66	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.519
67	antialias-resnet152	.512
68	efficientnet-b7	.510
69	vgg_19	.505
70	resnet50_moco_v2	.504
71	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.499
72	Res2Net50_26w_4s	.491
73	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.491
74	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.489
75	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.488
76	densenet-201-keras	.479
77	resnet34_imagenet_full	.478
78	resnet-50-pytorch	.474
79	resnet50-sup	.474
80	resnet50_tutorial	.474
81	resnet_50_v1	.474
82	grcnn_109	.474
83	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.472
84	convnext_large:fb_in22k_ft_in1k	.470
85	AT_efficientnet-b2	.468
86	AdvProp_efficientnet-b2	.468
87	resnet50_ewconplus_wzc	.466
88	resnet50-SIN_IN_IN	.464
89	resnet_SIN_IN_FT_IN	.464
90	ViT_L_32_imagenet1k	.463
91	convnext_small_imagenet_full_seed-0	.462
92	resnet50-moclr8deg	.461
93	densenet-169	.456
94	deit_small_imagenet_full_seed-0	.454
95	AdvProp_efficientnet-b4	.450
96	AT_efficientnet-b4	.450
97	resnet_50_v1_spiking	.448
98	densenet-169-keras	.448
99	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.448
100	densenet-121-keras	.448
101	cornet_s	.445
102	resnet18_imagenet_full	.443
103	densenet_201_pytorch	.442
104	convnext_xlarge:fb_in22k_ft_in1k	.441
105	grcnn	.439
106	densenet-201	.438
107	resnet-101_v1-tf	.432
108	mobilenet_v2_1_3_224	.430
109	efficientnet_b2_imagenet_full	.429
110	vit_large_patch14_clip_224:openai_ft_in1k	.428
111	resnet50_random_linf8_perturb	.427
112	convnext_base:clip_laiona_augreg_ft_in1k_384	.422
113	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.419
114	deit_large_imagenet_full_seed-0	.411
115	convnext_small_imagenet_100_seed-0	.406
116	deit_base_imagenet_full_seed-0	.406
117	densenet-121	.404
118	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.403
119	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.403
120	cv_18_dagger_408_pretrained	.400
121	resnet-50_v1-tf	.395
122	convnext_large_imagenet_full_seed-0	.395
123	BiT-S-R50x3	.391
124	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.390
125	vit_large_patch14_clip_336:laion2b_ft_in1k	.387
126	ReAlnet03	.377
127	resnet-152_v1-tf	.373
128	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.370
129	ReAlnet01	.370
130	convnext_base_imagenet_full_seed-0	.365
131	mobilenet_v2_1.4_224-tf	.363
132	voneresnet-50-non_stochastic	.361
133	BiT-S-R152x2	.356
134	custom_model_cv_18_dagger_408	.346
135	resnet50-SIN_IN	.344
136	inception_v2-tf	.344
137	mobilenet_v2_1.3_224-tf	.343
138	ReAlnet04	.342
139	vgg_16	.341
140	ReAlnet08	.328
141	ReAlnet09	.325
142	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.324
143	ReAlnet05	.322
144	efficientnet_b0_imagenet_full	.320
145	ViT_B_32_imagenet1k	.320
146	ReAlnet07	.316
147	CORnet-S	.314
148	ReAlnet06	.314
149	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.311
150	nasnet_mobile-tf	.309
151	inception_v1	.300
152	ReAlnet02	.299
153	vonegrcnn_62e_nobn	.298
154	vonegrcnn_62e_nobn	.293
155	efficientnet_b1_imagenet_full	.290
156	ReAlnet10	.290
157	mobilenet_v2_1-4_224_pytorch	.288
158	mobilenet_v2_1_4_224	.288
159	inception_v1-tf	.288
160	nasnet_mobile	.286
161	vonegrcnn_47e	.284
162	BiT-S-R50x1	.269
163	resnet50_linf_4_robust	.268
164	AdvProp_efficientnet-b8	.263
165	BiT-S-R101x3	.263
166	BiT-S-R152x4	.261
167	resnet-18	.260
168	vonegrcnn_47e	.260
169	convnext_tiny_sup	.254
170	mobilenet_v2_0_5_224	.253
171	resnet-34	.252
172	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.248
173	regnet_y_400mf	.248
174	mobilenet_v2_1_0_192	.246
175	vonegrcnn_52e_full	.239
176	vonegrcnn_52e_full	.236
177	mobilenet_v2_0_75_224	.231
178	mobilenet_v2_1_0_160	.228
179	voneresnet-50	.218
180	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.216
181	mobilenet_v1_1.0_224	.215
182	blt_vs	.214
183	mobilenet_v2_1_0_224	.214
184	vgg-16-keras	.211
185	mobilenet_v1_0.75_224	.211
186	vgg-19-keras	.188
187	BiT-S-R101x1	.182
188	texture_shape_resnet50_trained_on_SIN	.182
189	voneresnet-50-robust	.180
190	resnet50_eMMCR_Vanilla	.179
191	mobilenet_v2_1.0_160-tf	.177
192	mobilenet_v1_1.0_160	.173
193	resnet50-vitoimagevidnet8	.170
194	mobilenet_v2_1.0_224-tf	.168
195	mobilenet_v2_0_5_192	.165
196	alexnet	.163
197	alexnet_random_linf8_perturb	.162
198	mobilenet_v2_1.0_192-tf	.161
199	mobilenet_v2_0.75_224-tf	.159
200	alexnet_random_l2_3_perturb	.152
201	resnet50_robust_l2_eps1	.152
202	mobilenet_v2_0.5_224-tf	.150
203	mobilenet_v2_0_75_192	.150
204	resnet50_l2_3_robust	.146
205	resnet_50_v1_spiking_l4a	.143
206	resnet50_simclr	.143
207	alexnet_reduced_aliasing_early_checkpoint	.137
208	mobilenet_v2_0_75_160	.134
209	alexnet-baseline	.134
210	alexnet	.134
211	alexnet	.134
212	alexnet	.134
213	alexnet	.134
214	alexnet	.134
215	alexnet	.134
216	alexnet	.134
217	alexnet	.134
218	alexnet	.134
219	alexnet_ks_torevert	.134
220	konkle_alexnetgn_ipcl_ref01_primary_model	.134
221	mobilenet_v2_0.5_192-tf	.131
222	shufflenet_v2_x1_0	.130
223	mobilenet_v1_0.5_224	.123
224	mobilenet_v2_1_0_128	.122
225	resnet50_imagenet_100_seed-0	.120
226	alexnet_early_checkpoint	.119
227	AdvProp_efficientnet-b0	.115
228	mobilenet_v1_0.75_192	.113
229	resnet50-SIN	.111
230	mobilenet_v2_0.75_160-tf	.108
231	vonealexnet_gaussian_noise_std4_fixed	.104
232	resnet-50-robust	.104
233	imagenet_l2_3_0	.104
234	resnet50_robust_l2_eps3	.104
235	resnet50_linf_8_robust	.104
236	mobilenet_v1_1.0_192	.096
237	mobilenet_v1_0.5_192	.090
238	0.5x_resnet-18_LC_w_sh_1_iter	.084
239	mobilenet_v2_0.75_192-tf	.072
240	mobilenet_v2_0.75_96	.072
241	texture_shape_alexnet_trained_on_SIN	.071
242	mobilenet_v1_0.75_160	.068
243	mobilenet_v1_1.0_128	.068
244	squeezenet1_0	.067
245	AlexNet_SIN	.065
246	mobilenet_v2_0.75_128	.062
247	mobilenet_v2_1.0_128-tf	.061
248	alexnet_linf_8_robust	.060
249	alexnet_l2_3_robust	.060
250	mobilenet_v2_0.35_160	.058
251	alexnet_training_seed_07	.050
252	alexnet_training_seed_02	.049
253	alexnet_training_seed_04	.049
254	alexnet_training_seed_08	.049
255	alexnet_training_seed_10	.049
256	mobilenet_v2_0.35_192	.048
257	squeezenet1_1	.047
258	mobilenet_v1_0.25_192	.045
259	mobilenet_v1_0.25_224	.044
260	alexnet_robust_correct	.043
261	resnet50-vicreg	.043
262	imagenet_l2_10_0	.041
263	mobilenet_v1_0.75_128	.039
264	mobilenet_v2_0.35_128	.038
265	mobilenet_v2_0.5_128	.037
266	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.037
267	mobilenet_v1_0.5_160	.036
268	mobilenet_v2_0.35_96	.036
269	mobilenet_v2_0.35_224	.035
270	deit_small_imagenet_100_seed-0	.035
271	resnet50-vicregl0p9	.031
272	mobilenet_v1_0.25_128	.030
273	mobilenet_v2_1.0_96	.030
274	convnext_small_imagenet_10_seed-0	.030
275	deit_small_imagenet_10_seed-0	.027
276	resnet50_imagenet_10_seed-0	.027
277	deit_small_imagenet_1_seed-0	.027
278	alexnet_training_seed_03	.025
279	mobilenet_v2_0.5_160	.025
280	mobilenet_v2_0.5_96	.025
281	resnet50-meshes-lt-100-original-pretrained	.022
282	alexnet_training_seed_06	.022
283	resnet50-barlow	.020
284	mobilenet_v1_0.5_128	.020
285	alexnet_training_seed_05	.020
286	alexnet_training_seed_01	.020
287	resnet_50_v1_spiking_l4	.019
288	artResNet18_1	.015
289	my-model	.014
290	resnet50-vicregl0p75	.014
291	resnet_50_v1_spiking_l3	.012
292	barlow-twins-resnet50	.009
293	resnet-50_untrained	.009
294	resnet50-meshes-lt-100-original-scratch	.009
295	convnext_small_imagenet_1_seed-0	.009
296	alexnet_training_seed_09	.007
297	mobilenet_v1_0.25_160	.006
298	resnet50_FractalDB	.004
299	mobilevit_small	.004
300	resnet50_imagenet_1_seed-0	.003
301	resnet50-simclr-vissl	.002
302	resnet-18-LC_w_sh_100_iter_conv_init_m
303	ViT-B/32
304	BiT-M-R50x3
305	BiT-M-R152x4
306	BiT-M-R50x1
307	BiT-M-R152x2
308	resnet-18-LC_w_sh_1_iter_conv_init_m
309	resnet-18-LC_w_sh_1_iter_m
310	resnet-18-LC_w_sh_10_iter_m
311	resnet-18-LC_w_sh_100_iter_m
312	resnet-18-LC_w_sh_10_iter_conv_init_m
313	FrankRobWobv0
314	resnet18-local_aggregation
315	0.5x_resnet-18_LC
316	CORnetZ_CIFAR10
317	resnet18-colorization
318	resnet18-deepcluster
319	r3m_resnet18
320	resnet18-instance_recognition
321	CORnet-Z
322	my-model
323	vggface
324	0.5x_resnet-18_LC_w_sh_10_iter
325	r3m_resnet50
326	resnet18-relative_position
327	r3m_resnet34
328	resnet-18-LC_m
329	r3m_resnet50_nocrop
330	resnet18-supervised
331	resnet-18-LC_1st_conv_m
332	resnet-18-LC_conv_init_m
333	resnet18-contrastive_multiview
334	0.5x_resnet-18
335	RN50
336	0.5x_resnet-18_LC_w_sh_100_iter
337	RN50
338	resnet18-autoencoder
339	resnet18-simclr
340	ViT_L_32
341	ViT_B_32
342	resnet18-depth_prediction
343	ViT_B_16
344	r3m_resnet34_nocrop
345	resnet18-contrastive_predictive
346	omnivore_swinT
347	resnet-18-LC_untrained
348	resnet-18_untrained
349	resnet-50x2_untrained
350	dcgan
351	v1-pyr-nodown
352	unet_entire
353	CORnetZ_CIFAR10_bs32_20_04
354	pixels
355	dorinet_cornet_z
356	resnet50
357	openclip
358	resnet50-cifar
359	resnet50_pretrained_with_retinal_waves
360	resnet50_ImageNet
361	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4
362	yudixie_resnet50_random_0_240908
363	yudixie_resnet50_imagenet1kpret_0_240908
364	yudixie_resnet50_cat_obj_class_all_latents_0_240908
365	yudixie_resnet50_object_class_0_240908
366	yudixie_resnet50_category_class_0_240908
367	resnet18_imagenet21kP
368	ConvLSTM
369	PredRNN
370	yudixie_resnet50_distance_reg_0_240908
371	SimVP
372	TAU
373	MIM
374	hmax
375	ReAlnet01_cornet
376	pnasnet_large_pytorch
377	inception_v3_pytorch
378	briaai_rmbg_1_4
379	yudixie_resnet50_imagenet1kpret_0_240312
380	yudixie_resnet18_translation_reg_0_240719
381	yudixie_resnet18_rotation_reg_0_240719
382	compact_resnet50_V4
383	yudixie_resnet18_imagenet1kpret_0_240719
384	ReAlnet02_cornet
385	resnet18_ecoset_full
386	compact_resnet50_IT
387	resnet50_ecoset_full
388	resnet101_ecoset_full
389	yudixie_resnet50_distance_rotation_0_240908
390	yudixie_resnet50_rotation_reg_0_240908
391	resnet152_ecoset_full
392	yudixie_resnet18_distance_translation_0_240719
393	ReAlnet04_cornet
394	yudixie_resnet50_distance_translation_0_240908
395	yudixie_resnet18_translation_rotation_0_240719
396	compact_resnet50_robust_V4
397	compact_vgg19_V4
398	ReAlnet05_cornet
399	yudixie_resnet18_random_0_240719
400	yudixie_resnet18_cat_obj_class_all_latents_0_240719
401	resnet-18_test_m
402	SeLaVi-AVE
403	yudixie_resnet18_distance_reg_0_240719
404	r3d_18
405	ReAlnet06_cornet
406	yudixie_resnet50_translation_reg_0_240908
407	compact_vgg19_IT
408	ReAlnet01_ALL
409	mc3_18
410	ReAlnet07_cornet
411	yudixie_resnet50_translation_rotation_0_240908
412	s3d
413	mvit_v1_b
414	ReAlnet08_cornet
415	compact_cornets_V4
416	ReAlnet02_ALL
417	ReAlnet03_ALL
418	VOneCORnet-S
419	ReAlnet04_ALL
420	omnivore_swinB
421	SeLaVi-Kinetics400
422	ReAlnet09_cornet
423	SeLaVi-VGG-Sound
424	compact_cornets_IT
425	ReAlnet05_ALL
426	SeLaVi-Kinetics-Sound
427	omnivore_swinS
428	ReAlnet06_ALL
429	mvit_v2_s
430	compact_resnet50_robust_IT
431	r2plus1d_18
432	ReAlnet07_ALL
433	resnet34_ecoset_full
434	yudixie_resnet18_object_class_0_240719
435	yudixie_resnet18_category_class_0_240719
436	ReAlnet08_ALL
437	yudixie_resnet50_distance_translation_rotation_0_240908
438	yudixie_resnet18_distance_translation_rotation_0_240719
439	yudixie_resnet18_distance_rotation_0_240719
440	ReAlnet03_cornet
441	ReAlnet09_ALL
442	ReAlnet10_cornet
443	resnet50_primary_visual_cortex
444	ReAlnet10_ALL
445	yolos_tiny

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.42.

Note that scores are relative to this ceiling.

Data: Geirhos2021colour

Metric: error_consistency