Scores on benchmarks
Model rank shown below is with respect to all public models.| .08 |
average_vision
rank 434
81 benchmarks |
|
| .01 |
neural_vision
rank 482
38 benchmarks |
|
| .06 |
V1
rank 449
24 benchmarks |
|
| .18 |
Coggan2024_fMRI.V1-rdm
v1
rank 31
|
|
|
||
| .14 |
behavior_vision
rank 225
43 benchmarks |
|
| .10 |
Geirhos2021-error_consistency
[reference]
rank 221
17 benchmarks |
|
| .18 |
Geirhos2021colour-error_consistency
v1
[reference]
rank 189
|
|
|
||
| .08 |
Geirhos2021contrast-error_consistency
v1
[reference]
rank 234
|
|
|
||
| .16 |
Geirhos2021cueconflict-error_consistency
v1
[reference]
rank 154
|
|
|
||
| .08 |
Geirhos2021edge-error_consistency
v1
[reference]
rank 134
|
|
|
||
| .08 |
Geirhos2021eidolonI-error_consistency
v1
[reference]
rank 283
|
|
|
||
| .17 |
Geirhos2021eidolonII-error_consistency
v1
[reference]
rank 227
|
|
|
||
| .12 |
Geirhos2021eidolonIII-error_consistency
v1
[reference]
rank 245
|
|
|
||
| .22 |
Geirhos2021falsecolour-error_consistency
v1
[reference]
rank 164
|
|
|
||
| .05 |
Geirhos2021highpass-error_consistency
v1
[reference]
rank 153
|
|
|
||
| .09 |
Geirhos2021lowpass-error_consistency
v1
[reference]
rank 193
|
|
|
||
| .05 |
Geirhos2021phasescrambling-error_consistency
v1
[reference]
rank 223
|
|
|
||
| .01 |
Geirhos2021powerequalisation-error_consistency
v1
[reference]
rank 286
|
|
|
||
| .06 |
Geirhos2021rotation-error_consistency
v1
[reference]
rank 206
|
|
|
||
| .05 |
Geirhos2021silhouette-error_consistency
v1
[reference]
rank 298
|
|
|
||
| .04 |
Geirhos2021sketch-error_consistency
v1
[reference]
rank 225
|
|
|
||
| .16 |
Geirhos2021stylized-error_consistency
v1
[reference]
rank 185
|
|
|
||
| .06 |
Geirhos2021uniformnoise-error_consistency
v1
[reference]
rank 188
|
|
|
||
| .09 |
Baker2022
rank 157
3 benchmarks |
|
| .27 |
Baker2022fragmented-accuracy_delta
v1
[reference]
rank 118
|
|
|
||
| .00 |
Baker2022frankenstein-accuracy_delta
v1
[reference]
rank 149
|
|
|
||
| .00 |
Baker2022inverted-accuracy_delta
v1
[reference]
rank 58
|
|
|
||
| .15 |
BMD2024
rank 124
4 benchmarks |
|
| .17 |
BMD2024.dotted_1Behavioral-accuracy_distance
v1
rank 87
|
|
|
||
| .13 |
BMD2024.dotted_2Behavioral-accuracy_distance
v1
rank 113
|
|
|
||
| .13 |
BMD2024.texture_1Behavioral-accuracy_distance
v1
rank 133
|
|
|
||
| .16 |
BMD2024.texture_2Behavioral-accuracy_distance
v1
rank 123
|
|
|
||
| .02 |
Ferguson2024
[reference]
rank 255
14 benchmarks |
|
| .27 |
Ferguson2024gray_hard-value_delta
v1
[reference]
rank 180
|
|
|
||
| .23 |
Hebart2023-match
v1
rank 150
|
|
|
||
| .42 |
Maniquet2024
rank 195
2 benchmarks |
|
| .16 |
Maniquet2024-confusion_similarity
v1
[reference]
rank 231
|
|
|
||
| .67 |
Maniquet2024-tasks_consistency
v1
[reference]
rank 79
|
|
|
||
| .10 |
Coggan2024_behavior-ConditionWiseAccuracySimilarity
v1
rank 177
|
|
|
||
| .10 |
engineering_vision
rank 286
25 benchmarks |
|
| .52 |
Geirhos2021-top1
[reference]
rank 154
17 benchmarks |
|
| .95 |
Geirhos2021colour-top1
v1
[reference]
rank 145
|
|
|
||
| .86 |
Geirhos2021contrast-top1
v1
[reference]
rank 88
|
|
|
||
| .16 |
Geirhos2021cueconflict-top1
v1
[reference]
rank 242
|
|
|
||
| .19 |
Geirhos2021edge-top1
v1
[reference]
rank 214
|
|
|
||
| .50 |
Geirhos2021eidolonI-top1
v1
[reference]
rank 133
|
|
|
||
| .51 |
Geirhos2021eidolonII-top1
v1
[reference]
rank 141
|
|
|
||
| .50 |
Geirhos2021eidolonIII-top1
v1
[reference]
rank 155
|
|
|
||
| .93 |
Geirhos2021falsecolour-top1
v1
[reference]
rank 127
|
|
|
||
| .50 |
Geirhos2021highpass-top1
v1
[reference]
rank 64
|
|
|
||
| .42 |
Geirhos2021lowpass-top1
v1
[reference]
rank 129
|
|
|
||
| .54 |
Geirhos2021phasescrambling-top1
v1
[reference]
rank 188
|
|
|
||
| .60 |
Geirhos2021powerequalisation-top1
v1
[reference]
rank 185
|
|
|
||
| .67 |
Geirhos2021rotation-top1
v1
[reference]
rank 128
|
|
|
||
| .32 |
Geirhos2021silhouette-top1
v1
[reference]
rank 244
|
|
|
||
| .49 |
Geirhos2021sketch-top1
v1
[reference]
rank 216
|
|
|
||
| .33 |
Geirhos2021stylized-top1
v1
[reference]
rank 206
|
|
|
||
| .38 |
Geirhos2021uniformnoise-top1
v1
[reference]
rank 151
|
|
|
||
How to use
from brainscore_vision import load_model
model = load_model("resnet50_eMMCR_Vanilla")
model.start_task(...)
model.start_recording(...)
model.look_at(...)
Benchmarks bibtex
@inproceedings{santurkar2019computer,
title={Computer Vision with a Single (Robust) Classifier},
author={Shibani Santurkar and Dimitris Tsipras and Brandon Tran and Andrew Ilyas and Logan Engstrom and Aleksander Madry},
booktitle={ArXiv preprint arXiv:1906.09453},
year={2019}
}
@article{geirhos2021partial,
title={Partial success in closing the gap between human and machine vision},
author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
journal={Advances in Neural Information Processing Systems},
volume={34},
year={2021},
url={https://openreview.net/forum?id=QkljT4mrfs}
}
@article{BAKER2022104913,
title = {Deep learning models fail to capture the configural nature of human shape perception},
journal = {iScience},
volume = {25},
number = {9},
pages = {104913},
year = {2022},
issn = {2589-0042},
doi = {https://doi.org/10.1016/j.isci.2022.104913},
url = {https://www.sciencedirect.com/science/article/pii/S2589004222011853},
author = {Nicholas Baker and James H. Elder},
keywords = {Biological sciences, Neuroscience, Sensory neuroscience},
abstract = {Summary
A hallmark of human object perception is sensitivity to the holistic configuration of the local shape features of an object. Deep convolutional neural networks (DCNNs) are currently the dominant models for object recognition processing in the visual cortex, but do they capture this configural sensitivity? To answer this question, we employed a dataset of animal silhouettes and created a variant of this dataset that disrupts the configuration of each object while preserving local features. While human performance was impacted by this manipulation, DCNN performance was not, indicating insensitivity to object configuration. Modifications to training and architecture to make networks more brain-like did not lead to configural processing, and none of the networks were able to accurately predict trial-by-trial human object judgements. We speculate that to match human configural sensitivity, networks must be trained to solve a broader range of object tasks beyond category recognition.}
}
@misc{ferguson_ngo_lee_dicarlo_schrimpf_2024,
title={How Well is Visual Search Asymmetry predicted by a Binary-Choice, Rapid, Accuracy-based Visual-search, Oddball-detection (BRAVO) task?},
url={osf.io/5ba3n},
DOI={10.17605/OSF.IO/5BA3N},
publisher={OSF},
author={Ferguson, Michael E, Jr and Ngo, Jerry and Lee, Michael and DiCarlo, James and Schrimpf, Martin},
year={2024},
month={Jun}
}
@article {Maniquet2024.04.02.587669,
author = {Maniquet, Tim and de Beeck, Hans Op and Costantino, Andrea Ivan},
title = {Recurrent issues with deep neural network models of visual recognition},
elocation-id = {2024.04.02.587669},
year = {2024},
doi = {10.1101/2024.04.02.587669},
publisher = {Cold Spring Harbor Laboratory},
URL = {https://www.biorxiv.org/content/early/2024/04/10/2024.04.02.587669},
eprint = {https://www.biorxiv.org/content/early/2024/04/10/2024.04.02.587669.full.pdf},
journal = {bioRxiv}
}