PG v1.1 - Initial commit

.gitignore

@@ -0,0 +1,74 @@
+# Temporarily ignore CSVs until we choose which ones should be committed
+*.csv
+slurm
+archives
+output
+__pycache__
+*.pyc
+.ipynb_checkpoints
+*.html
+
+# From GitHub .gitignore Python template
+
+# Ignore large files, download manually
+# data/reference_files/
+# data/
+alignments
+baseline_models/model_checkpoints/
+outputs/intermediate_outputs/
+outputs/output_scores/
+clustal/
+processed_data/
+make_website_data_folder.py
+website_data/
+
+# Ignore vespa output
+vespa_run_directory
+
+# Ignoring slurm scripts that we use for scoring
+o2_scripts/
+slurm_scripts/
+
+# Ignoring HMM model files
+# And temporary scripts directories
+scripts/scoring_DMS_zero_shot/HMM_temp/
+scripts/scoring_DMS_zero_shot/gemme_tmp/
+**/gemme_tmp/*
+**/vespa_tmp/*
+
+
+# Optional:
+baseline_models/bash_scripts/old/
+# Slurm script outputs
+*.out
+*.err
+logs/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Jupyter Notebook
+.ipynb_checkpoints

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 OATML-Markslab, Pascal Notin, Aaron Kollasch, Daniel Ritter, Lood van Niekerk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

benchmarks/DMS_supervised/indels/MSE/DMS_indels_MSE_DMS_level.csv

@@ -0,0 +1,67 @@
+DMS_id,ESM-1v Embeddings,MSA Transformer Embeddings,Tranception Embeddings
+B1LPA6_ECOSM_Russ_2020_indels,0.547,0.569,0.793
+BLAT_ECOLX_Gonzalez_2019_indels,0.366,0.313,0.273
+CAPSD_AAV2S_Sinai_2021_designed_indels,0.216,0.205,0.25
+CAPSD_AAV2S_Sinai_2021_library_indels,0.434,0.414,0.299
+HIS7_YEAST_Pokusaeva_2019_indels,0.161,0.198,0.158
+PTEN_HUMAN_Mighell_2018_indels,0.274,0.467,0.365
+P53_HUMAN_Kotler_2018_indels,0.315,0.383,0.348
+KCNJ2_MOUSE_Macdonald_2022_indels,0.517,0.671,0.391
+Q8EG35_SHEON_Campbell_2022_indels,0.57,0.624,0.512
+A4_HUMAN_Seuma_2022_indels,0.262,0.36,0.249
+S22A1_HUMAN_Yee_2023_abundance_indels,0.42,0.613,0.44
+S22A1_HUMAN_Yee_2023_activity_indels,0.427,0.606,0.397
+AMFR_HUMAN_Rocklin_2023_4G3O_indels,0.267,0.467,0.848
+ARGR_ECOLI_Rocklin_2023_1AOY_indels,0.138,0.259,0.204
+BBC1_YEAST_Rocklin_2023_1TG0_indels,0.224,0.328,0.391
+BCHB_CHLTE_Rocklin_2023_2KRU_indels,0.194,0.503,0.449
+CATR_CHLRE_Rocklin_2023_2AMI_indels,0.192,0.362,0.247
+CBPA2_HUMAN_Rocklin_2023_1O6X_indels,0.131,0.292,0.346
+CBX4_HUMAN_Rocklin_2023_2K28_indels,0.246,0.423,0.409
+CSN4_MOUSE_Rocklin_2023_1UFM_indels,0.087,0.292,0.288
+CUE1_YEAST_Rocklin_2023_2MYX_indels,0.216,0.361,0.56
+DN7A_SACS2_Rocklin_2023_1JIC_indels,0.282,0.547,0.549
+DNJA1_HUMAN_Rocklin_2023_2LO1_indels,0.128,0.25,0.162
+DOCK1_MOUSE_Rocklin_2023_2M0Y_indels,0.281,0.491,0.353
+EPHB2_HUMAN_Rocklin_2023_1F0M_indels,0.125,0.299,0.307
+FECA_ECOLI_Rocklin_2023_2D1U_indels,0.488,0.508,0.592
+HCP_LAMBD_Rocklin_2023_2L6Q_indels,0.091,0.267,0.238
+HECD1_HUMAN_Rocklin_2023_3DKM_indels,0.63,0.568,0.645
+ILF3_HUMAN_Rocklin_2023_2L33_indels,0.135,0.304,0.245
+MAFG_MOUSE_Rocklin_2023_1K1V_indels,0.257,0.548,0.493
+MBD11_ARATH_Rocklin_2023_6ACV_indels,0.272,0.377,0.597
+MYO3_YEAST_Rocklin_2023_2BTT_indels,0.392,0.612,0.514
+NKX31_HUMAN_Rocklin_2023_2L9R_indels,0.153,0.367,0.156
+NUSA_ECOLI_Rocklin_2023_1WCL_indels,0.154,0.284,0.527
+NUSG_MYCTU_Rocklin_2023_2MI6_indels,0.22,0.388,0.327
+OBSCN_HUMAN_Rocklin_2023_1V1C_indels,0.178,0.282,0.342
+ODP2_GEOSE_Rocklin_2023_1W4G_indels,0.46,0.682,0.543
+OTU7A_HUMAN_Rocklin_2023_2L2D_indels,0.216,0.422,0.56
+PIN1_HUMAN_Rocklin_2023_1I6C_indels,0.113,0.389,0.269
+PITX2_HUMAN_Rocklin_2023_2L7M_indels,0.495,0.773,0.605
+PKN1_HUMAN_Rocklin_2023_1URF_indels,0.082,0.178,0.156
+POLG_PESV_Rocklin_2023_2MXD_indels,0.198,0.495,0.384
+PR40A_HUMAN_Rocklin_2023_1UZC_indels,0.168,0.312,0.279
+PSAE_SYNP2_Rocklin_2023_1PSE_indels,0.24,0.259,0.311
+RAD_ANTMA_Rocklin_2023_2CJJ_indels,0.193,0.33,0.294
+RCD1_ARATH_Rocklin_2023_5OAO_indels,0.251,0.359,0.424
+RD23A_HUMAN_Rocklin_2023_1IFY_indels,0.154,0.349,0.29
+RPC1_BP434_Rocklin_2023_1R69_indels,0.484,0.511,0.431
+RS15_GEOSE_Rocklin_2023_1A32_indels,0.203,0.355,0.331
+SAV1_MOUSE_Rocklin_2023_2YSB_indels,0.531,0.708,0.648
+SDA_BACSU_Rocklin_2023_1PV0_indels,0.113,0.197,0.184
+SOX30_HUMAN_Rocklin_2023_7JJK_indels,0.195,0.336,0.566
+SPG2_STRSG_Rocklin_2023_5UBS_indels,0.16,0.256,0.262
+SPTN1_CHICK_Rocklin_2023_1TUD_indels,0.237,0.332,0.471
+SQSTM_MOUSE_Rocklin_2023_2RRU_indels,0.225,0.559,0.442
+SR43C_ARATH_Rocklin_2023_2N88_indels,0.223,0.323,0.235
+SRBS1_HUMAN_Rocklin_2023_2O2W_indels,0.153,0.366,0.201
+TCRG1_MOUSE_Rocklin_2023_1E0L_indels,0.118,0.22,0.329
+THO1_YEAST_Rocklin_2023_2WQG_indels,0.233,0.421,0.623
+TNKS2_HUMAN_Rocklin_2023_5JRT_indels,0.22,0.405,0.477
+UBE4B_HUMAN_Rocklin_2023_3L1X_indels,0.348,0.431,0.533
+UBR5_HUMAN_Rocklin_2023_1I2T_indels,0.196,0.344,0.288
+VG08_BPP22_Rocklin_2023_2GP8_indels,0.187,0.291,0.394
+VILI_CHICK_Rocklin_2023_1YU5_indels,0.133,0.215,0.184
+VRPI_BPT7_Rocklin_2023_2WNM_indels,0.235,0.496,0.474
+YNZC_BACSU_Rocklin_2023_2JVD_indels,0.1,0.343,0.303

benchmarks/DMS_supervised/indels/MSE/DMS_indels_MSE_DMS_level_fold_random_5.csv

@@ -0,0 +1,67 @@
+DMS_id,ESM-1v Embeddings,MSA Transformer Embeddings,Tranception Embeddings
+B1LPA6_ECOSM_Russ_2020_indels,0.547,0.569,0.793
+BLAT_ECOLX_Gonzalez_2019_indels,0.366,0.313,0.273
+CAPSD_AAV2S_Sinai_2021_designed_indels,0.216,0.205,0.25
+CAPSD_AAV2S_Sinai_2021_library_indels,0.434,0.414,0.299
+HIS7_YEAST_Pokusaeva_2019_indels,0.161,0.198,0.158
+PTEN_HUMAN_Mighell_2018_indels,0.274,0.467,0.365
+P53_HUMAN_Kotler_2018_indels,0.315,0.383,0.348
+KCNJ2_MOUSE_Macdonald_2022_indels,0.517,0.671,0.391
+Q8EG35_SHEON_Campbell_2022_indels,0.57,0.624,0.512
+A4_HUMAN_Seuma_2022_indels,0.262,0.36,0.249
+S22A1_HUMAN_Yee_2023_abundance_indels,0.42,0.613,0.44
+S22A1_HUMAN_Yee_2023_activity_indels,0.427,0.606,0.397
+AMFR_HUMAN_Rocklin_2023_4G3O_indels,0.267,0.467,0.848
+ARGR_ECOLI_Rocklin_2023_1AOY_indels,0.138,0.259,0.204
+BBC1_YEAST_Rocklin_2023_1TG0_indels,0.224,0.328,0.391
+BCHB_CHLTE_Rocklin_2023_2KRU_indels,0.194,0.503,0.449
+CATR_CHLRE_Rocklin_2023_2AMI_indels,0.192,0.362,0.247
+CBPA2_HUMAN_Rocklin_2023_1O6X_indels,0.131,0.292,0.346
+CBX4_HUMAN_Rocklin_2023_2K28_indels,0.246,0.423,0.409
+CSN4_MOUSE_Rocklin_2023_1UFM_indels,0.087,0.292,0.288
+CUE1_YEAST_Rocklin_2023_2MYX_indels,0.216,0.361,0.56
+DN7A_SACS2_Rocklin_2023_1JIC_indels,0.282,0.547,0.549
+DNJA1_HUMAN_Rocklin_2023_2LO1_indels,0.128,0.25,0.162
+DOCK1_MOUSE_Rocklin_2023_2M0Y_indels,0.281,0.491,0.353
+EPHB2_HUMAN_Rocklin_2023_1F0M_indels,0.125,0.299,0.307
+FECA_ECOLI_Rocklin_2023_2D1U_indels,0.488,0.508,0.592
+HCP_LAMBD_Rocklin_2023_2L6Q_indels,0.091,0.267,0.238
+HECD1_HUMAN_Rocklin_2023_3DKM_indels,0.63,0.568,0.645
+ILF3_HUMAN_Rocklin_2023_2L33_indels,0.135,0.304,0.245
+MAFG_MOUSE_Rocklin_2023_1K1V_indels,0.257,0.548,0.493
+MBD11_ARATH_Rocklin_2023_6ACV_indels,0.272,0.377,0.597
+MYO3_YEAST_Rocklin_2023_2BTT_indels,0.392,0.612,0.514
+NKX31_HUMAN_Rocklin_2023_2L9R_indels,0.153,0.367,0.156
+NUSA_ECOLI_Rocklin_2023_1WCL_indels,0.154,0.284,0.527
+NUSG_MYCTU_Rocklin_2023_2MI6_indels,0.22,0.388,0.327
+OBSCN_HUMAN_Rocklin_2023_1V1C_indels,0.178,0.282,0.342
+ODP2_GEOSE_Rocklin_2023_1W4G_indels,0.46,0.682,0.543
+OTU7A_HUMAN_Rocklin_2023_2L2D_indels,0.216,0.422,0.56
+PIN1_HUMAN_Rocklin_2023_1I6C_indels,0.113,0.389,0.269
+PITX2_HUMAN_Rocklin_2023_2L7M_indels,0.495,0.773,0.605
+PKN1_HUMAN_Rocklin_2023_1URF_indels,0.082,0.178,0.156
+POLG_PESV_Rocklin_2023_2MXD_indels,0.198,0.495,0.384
+PR40A_HUMAN_Rocklin_2023_1UZC_indels,0.168,0.312,0.279
+PSAE_SYNP2_Rocklin_2023_1PSE_indels,0.24,0.259,0.311
+RAD_ANTMA_Rocklin_2023_2CJJ_indels,0.193,0.33,0.294
+RCD1_ARATH_Rocklin_2023_5OAO_indels,0.251,0.359,0.424
+RD23A_HUMAN_Rocklin_2023_1IFY_indels,0.154,0.349,0.29
+RPC1_BP434_Rocklin_2023_1R69_indels,0.484,0.511,0.431
+RS15_GEOSE_Rocklin_2023_1A32_indels,0.203,0.355,0.331
+SAV1_MOUSE_Rocklin_2023_2YSB_indels,0.531,0.708,0.648
+SDA_BACSU_Rocklin_2023_1PV0_indels,0.113,0.197,0.184
+SOX30_HUMAN_Rocklin_2023_7JJK_indels,0.195,0.336,0.566
+SPG2_STRSG_Rocklin_2023_5UBS_indels,0.16,0.256,0.262
+SPTN1_CHICK_Rocklin_2023_1TUD_indels,0.237,0.332,0.471
+SQSTM_MOUSE_Rocklin_2023_2RRU_indels,0.225,0.559,0.442
+SR43C_ARATH_Rocklin_2023_2N88_indels,0.223,0.323,0.235
+SRBS1_HUMAN_Rocklin_2023_2O2W_indels,0.153,0.366,0.201
+TCRG1_MOUSE_Rocklin_2023_1E0L_indels,0.118,0.22,0.329
+THO1_YEAST_Rocklin_2023_2WQG_indels,0.233,0.421,0.623
+TNKS2_HUMAN_Rocklin_2023_5JRT_indels,0.22,0.405,0.477
+UBE4B_HUMAN_Rocklin_2023_3L1X_indels,0.348,0.431,0.533
+UBR5_HUMAN_Rocklin_2023_1I2T_indels,0.196,0.344,0.288
+VG08_BPP22_Rocklin_2023_2GP8_indels,0.187,0.291,0.394
+VILI_CHICK_Rocklin_2023_1YU5_indels,0.133,0.215,0.184
+VRPI_BPT7_Rocklin_2023_2WNM_indels,0.235,0.496,0.474
+YNZC_BACSU_Rocklin_2023_2JVD_indels,0.1,0.343,0.303

benchmarks/DMS_supervised/indels/MSE/Summary_performance_DMS_indels_MSE.csv

@@ -0,0 +1,4 @@
+Model_rank,Model_name,Model type,Average_MSE,Bootstrap_standard_error_MSE,Average_MSE_fold_random_5,Function_Activity,Function_Binding,Function_Expression,Function_OrganismalFitness,Function_Stability,Low_MSA_depth,Medium_MSA_depth,High_MSA_depth,Taxa_Human,Taxa_Other_Eukaryote,Taxa_Prokaryote,Taxa_Virus,References,Model details
+1,ESM-1v Embeddings,Embedding,0.365,0.0,0.365,0.416,N/A,0.468,0.347,0.229,0.248,0.301,0.25,0.262,0.234,0.268,0.253,"[1] Original model: <a href='https://proceedings.neurips.cc/paper/2021/hash/f51338d736f95dd42427296047067694-Abstract.html'>Meier, J., Rao, R., Verkuil, R., Liu, J., Sercu, T., & Rives, A. (2021). Language models enable zero-shot prediction of the effects of mutations on protein function. NeurIPS.</a> [2] Extension: <a href='https://openreview.net/forum?id=AwzbQVuDBk'>Notin, P., Weitzman, R., Marks, D. S., & Gal, Y. (2023). ProteinNPT: Improving protein property prediction and design with non-parametric transformers. Thirty-Seventh Conference on Neural Information Processing Systems</a>",ESM-1v Embeddings
+2,Tranception Embeddings,Embedding,0.41,0.022,0.41,0.518,N/A,0.415,0.313,0.391,0.385,0.291,0.409,0.412,0.378,0.398,0.366,"[1] Original model: <a href='https://proceedings.mlr.press/v162/notin22a.html'>Notin, P., Dias, M., Frazer, J., Marchena-Hurtado, J., Gomez, A.N., Marks, D.S., & Gal, Y. (2022). Tranception: Protein Fitness Prediction with Autoregressive Transformers and Inference-time Retrieval. ICML.</a> [2] Extension: <a href='https://openreview.net/forum?id=AwzbQVuDBk'>Notin, P., Weitzman, R., Marks, D. S., & Gal, Y. (2023). ProteinNPT: Improving protein property prediction and design with non-parametric transformers. Thirty-Seventh Conference on Neural Information Processing Systems</a>",Tranception Embeddings
+3,MSA Transformer Embeddings,Embedding,0.486,0.013,0.486,0.547,N/A,0.642,0.366,0.389,0.396,0.351,0.407,0.411,0.391,0.399,0.395,"[1] Original model: <a href='http://proceedings.mlr.press/v139/rao21a.html'>Rao, R., Liu, J., Verkuil, R., Meier, J., Canny, J.F., Abbeel, P., Sercu, T., & Rives, A. (2021). MSA Transformer. ICML.</a> [2] Extension: <a href='https://openreview.net/forum?id=AwzbQVuDBk'>Notin, P., Weitzman, R., Marks, D. S., & Gal, Y. (2023). ProteinNPT: Improving protein property prediction and design with non-parametric transformers. Thirty-Seventh Conference on Neural Information Processing Systems</a>",MSA Transformer Embeddings

benchmarks/DMS_supervised/indels/Spearman/DMS_indels_Spearman_DMS_level.csv

@@ -0,0 +1,67 @@
+DMS_id,ESM-1v Embeddings,MSA Transformer Embeddings,Tranception Embeddings
+B1LPA6_ECOSM_Russ_2020_indels,0.591,0.581,0.536
+BLAT_ECOLX_Gonzalez_2019_indels,0.78,0.814,0.78
+CAPSD_AAV2S_Sinai_2021_designed_indels,0.834,0.832,0.828
+CAPSD_AAV2S_Sinai_2021_library_indels,0.656,0.655,0.708
+HIS7_YEAST_Pokusaeva_2019_indels,0.623,0.62,0.597
+PTEN_HUMAN_Mighell_2018_indels,0.821,0.767,0.789
+P53_HUMAN_Kotler_2018_indels,0.68,0.679,0.653
+KCNJ2_MOUSE_Macdonald_2022_indels,0.678,0.563,0.733
+Q8EG35_SHEON_Campbell_2022_indels,0.764,0.693,0.782
+A4_HUMAN_Seuma_2022_indels,0.816,0.746,0.813
+S22A1_HUMAN_Yee_2023_abundance_indels,0.779,0.664,0.773
+S22A1_HUMAN_Yee_2023_activity_indels,0.705,0.636,0.696
+AMFR_HUMAN_Rocklin_2023_4G3O_indels,0.757,0.636,0.621
+ARGR_ECOLI_Rocklin_2023_1AOY_indels,0.926,0.87,0.9
+BBC1_YEAST_Rocklin_2023_1TG0_indels,0.849,0.795,0.811
+BCHB_CHLTE_Rocklin_2023_2KRU_indels,0.881,0.718,0.776
+CATR_CHLRE_Rocklin_2023_2AMI_indels,0.914,0.847,0.89
+CBPA2_HUMAN_Rocklin_2023_1O6X_indels,0.937,0.847,0.846
+CBX4_HUMAN_Rocklin_2023_2K28_indels,0.874,0.786,0.782
+CSN4_MOUSE_Rocklin_2023_1UFM_indels,0.9,0.813,0.831
+CUE1_YEAST_Rocklin_2023_2MYX_indels,0.759,0.639,0.588
+DN7A_SACS2_Rocklin_2023_1JIC_indels,0.858,0.707,0.74
+DNJA1_HUMAN_Rocklin_2023_2LO1_indels,0.93,0.879,0.925
+DOCK1_MOUSE_Rocklin_2023_2M0Y_indels,0.836,0.719,0.823
+EPHB2_HUMAN_Rocklin_2023_1F0M_indels,0.907,0.828,0.81
+FECA_ECOLI_Rocklin_2023_2D1U_indels,0.729,0.679,0.677
+HCP_LAMBD_Rocklin_2023_2L6Q_indels,0.954,0.858,0.906
+HECD1_HUMAN_Rocklin_2023_3DKM_indels,0.708,0.721,0.673
+ILF3_HUMAN_Rocklin_2023_2L33_indels,0.938,0.858,0.893
+MAFG_MOUSE_Rocklin_2023_1K1V_indels,0.843,0.696,0.705
+MBD11_ARATH_Rocklin_2023_6ACV_indels,0.84,0.813,0.72
+MYO3_YEAST_Rocklin_2023_2BTT_indels,0.761,0.599,0.771
+NKX31_HUMAN_Rocklin_2023_2L9R_indels,0.853,0.746,0.86
+NUSA_ECOLI_Rocklin_2023_1WCL_indels,0.917,0.847,0.781
+NUSG_MYCTU_Rocklin_2023_2MI6_indels,0.831,0.712,0.768
+OBSCN_HUMAN_Rocklin_2023_1V1C_indels,0.913,0.867,0.85
+ODP2_GEOSE_Rocklin_2023_1W4G_indels,0.83,0.733,0.812
+OTU7A_HUMAN_Rocklin_2023_2L2D_indels,0.805,0.693,0.742
+PIN1_HUMAN_Rocklin_2023_1I6C_indels,0.839,0.754,0.793
+PITX2_HUMAN_Rocklin_2023_2L7M_indels,0.678,0.579,0.657
+PKN1_HUMAN_Rocklin_2023_1URF_indels,0.925,0.879,0.876
+POLG_PESV_Rocklin_2023_2MXD_indels,0.93,0.763,0.812
+PR40A_HUMAN_Rocklin_2023_1UZC_indels,0.893,0.827,0.843
+PSAE_SYNP2_Rocklin_2023_1PSE_indels,0.821,0.822,0.85
+RAD_ANTMA_Rocklin_2023_2CJJ_indels,0.87,0.799,0.847
+RCD1_ARATH_Rocklin_2023_5OAO_indels,0.861,0.798,0.771
+RD23A_HUMAN_Rocklin_2023_1IFY_indels,0.908,0.792,0.844
+RPC1_BP434_Rocklin_2023_1R69_indels,0.658,0.609,0.677
+RS15_GEOSE_Rocklin_2023_1A32_indels,0.93,0.838,0.846
+SAV1_MOUSE_Rocklin_2023_2YSB_indels,0.692,0.569,0.727
+SDA_BACSU_Rocklin_2023_1PV0_indels,0.946,0.907,0.92
+SOX30_HUMAN_Rocklin_2023_7JJK_indels,0.875,0.81,0.826
+SPG2_STRSG_Rocklin_2023_5UBS_indels,0.922,0.862,0.869
+SPTN1_CHICK_Rocklin_2023_1TUD_indels,0.865,0.806,0.81
+SQSTM_MOUSE_Rocklin_2023_2RRU_indels,0.859,0.675,0.773
+SR43C_ARATH_Rocklin_2023_2N88_indels,0.879,0.821,0.87
+SRBS1_HUMAN_Rocklin_2023_2O2W_indels,0.927,0.814,0.916
+TCRG1_MOUSE_Rocklin_2023_1E0L_indels,0.9,0.834,0.792
+THO1_YEAST_Rocklin_2023_2WQG_indels,0.833,0.687,0.698
+TNKS2_HUMAN_Rocklin_2023_5JRT_indels,0.852,0.746,0.726
+UBE4B_HUMAN_Rocklin_2023_3L1X_indels,0.733,0.647,0.584
+UBR5_HUMAN_Rocklin_2023_1I2T_indels,0.893,0.82,0.869
+VG08_BPP22_Rocklin_2023_2GP8_indels,0.851,0.824,0.78
+VILI_CHICK_Rocklin_2023_1YU5_indels,0.932,0.899,0.91
+VRPI_BPT7_Rocklin_2023_2WNM_indels,0.873,0.725,0.77
+YNZC_BACSU_Rocklin_2023_2JVD_indels,0.878,0.759,0.839

benchmarks/DMS_supervised/indels/Spearman/DMS_indels_Spearman_DMS_level_fold_random_5.csv

@@ -0,0 +1,67 @@
+DMS_id,ESM-1v Embeddings,MSA Transformer Embeddings,Tranception Embeddings
+B1LPA6_ECOSM_Russ_2020_indels,0.591,0.581,0.536
+BLAT_ECOLX_Gonzalez_2019_indels,0.78,0.814,0.78
+CAPSD_AAV2S_Sinai_2021_designed_indels,0.834,0.832,0.828
+CAPSD_AAV2S_Sinai_2021_library_indels,0.656,0.655,0.708
+HIS7_YEAST_Pokusaeva_2019_indels,0.623,0.62,0.597
+PTEN_HUMAN_Mighell_2018_indels,0.821,0.767,0.789
+P53_HUMAN_Kotler_2018_indels,0.68,0.679,0.653
+KCNJ2_MOUSE_Macdonald_2022_indels,0.678,0.563,0.733
+Q8EG35_SHEON_Campbell_2022_indels,0.764,0.693,0.782
+A4_HUMAN_Seuma_2022_indels,0.816,0.746,0.813
+S22A1_HUMAN_Yee_2023_abundance_indels,0.779,0.664,0.773
+S22A1_HUMAN_Yee_2023_activity_indels,0.705,0.636,0.696
+AMFR_HUMAN_Rocklin_2023_4G3O_indels,0.757,0.636,0.621
+ARGR_ECOLI_Rocklin_2023_1AOY_indels,0.926,0.87,0.9
+BBC1_YEAST_Rocklin_2023_1TG0_indels,0.849,0.795,0.811
+BCHB_CHLTE_Rocklin_2023_2KRU_indels,0.881,0.718,0.776
+CATR_CHLRE_Rocklin_2023_2AMI_indels,0.914,0.847,0.89
+CBPA2_HUMAN_Rocklin_2023_1O6X_indels,0.937,0.847,0.846
+CBX4_HUMAN_Rocklin_2023_2K28_indels,0.874,0.786,0.782
+CSN4_MOUSE_Rocklin_2023_1UFM_indels,0.9,0.813,0.831
+CUE1_YEAST_Rocklin_2023_2MYX_indels,0.759,0.639,0.588
+DN7A_SACS2_Rocklin_2023_1JIC_indels,0.858,0.707,0.74
+DNJA1_HUMAN_Rocklin_2023_2LO1_indels,0.93,0.879,0.925
+DOCK1_MOUSE_Rocklin_2023_2M0Y_indels,0.836,0.719,0.823
+EPHB2_HUMAN_Rocklin_2023_1F0M_indels,0.907,0.828,0.81
+FECA_ECOLI_Rocklin_2023_2D1U_indels,0.729,0.679,0.677
+HCP_LAMBD_Rocklin_2023_2L6Q_indels,0.954,0.858,0.906
+HECD1_HUMAN_Rocklin_2023_3DKM_indels,0.708,0.721,0.673
+ILF3_HUMAN_Rocklin_2023_2L33_indels,0.938,0.858,0.893
+MAFG_MOUSE_Rocklin_2023_1K1V_indels,0.843,0.696,0.705
+MBD11_ARATH_Rocklin_2023_6ACV_indels,0.84,0.813,0.72
+MYO3_YEAST_Rocklin_2023_2BTT_indels,0.761,0.599,0.771
+NKX31_HUMAN_Rocklin_2023_2L9R_indels,0.853,0.746,0.86
+NUSA_ECOLI_Rocklin_2023_1WCL_indels,0.917,0.847,0.781
+NUSG_MYCTU_Rocklin_2023_2MI6_indels,0.831,0.712,0.768
+OBSCN_HUMAN_Rocklin_2023_1V1C_indels,0.913,0.867,0.85
+ODP2_GEOSE_Rocklin_2023_1W4G_indels,0.83,0.733,0.812
+OTU7A_HUMAN_Rocklin_2023_2L2D_indels,0.805,0.693,0.742
+PIN1_HUMAN_Rocklin_2023_1I6C_indels,0.839,0.754,0.793
+PITX2_HUMAN_Rocklin_2023_2L7M_indels,0.678,0.579,0.657
+PKN1_HUMAN_Rocklin_2023_1URF_indels,0.925,0.879,0.876
+POLG_PESV_Rocklin_2023_2MXD_indels,0.93,0.763,0.812
+PR40A_HUMAN_Rocklin_2023_1UZC_indels,0.893,0.827,0.843
+PSAE_SYNP2_Rocklin_2023_1PSE_indels,0.821,0.822,0.85
+RAD_ANTMA_Rocklin_2023_2CJJ_indels,0.87,0.799,0.847
+RCD1_ARATH_Rocklin_2023_5OAO_indels,0.861,0.798,0.771
+RD23A_HUMAN_Rocklin_2023_1IFY_indels,0.908,0.792,0.844
+RPC1_BP434_Rocklin_2023_1R69_indels,0.658,0.609,0.677
+RS15_GEOSE_Rocklin_2023_1A32_indels,0.93,0.838,0.846
+SAV1_MOUSE_Rocklin_2023_2YSB_indels,0.692,0.569,0.727
+SDA_BACSU_Rocklin_2023_1PV0_indels,0.946,0.907,0.92
+SOX30_HUMAN_Rocklin_2023_7JJK_indels,0.875,0.81,0.826
+SPG2_STRSG_Rocklin_2023_5UBS_indels,0.922,0.862,0.869
+SPTN1_CHICK_Rocklin_2023_1TUD_indels,0.865,0.806,0.81
+SQSTM_MOUSE_Rocklin_2023_2RRU_indels,0.859,0.675,0.773
+SR43C_ARATH_Rocklin_2023_2N88_indels,0.879,0.821,0.87
+SRBS1_HUMAN_Rocklin_2023_2O2W_indels,0.927,0.814,0.916
+TCRG1_MOUSE_Rocklin_2023_1E0L_indels,0.9,0.834,0.792
+THO1_YEAST_Rocklin_2023_2WQG_indels,0.833,0.687,0.698
+TNKS2_HUMAN_Rocklin_2023_5JRT_indels,0.852,0.746,0.726
+UBE4B_HUMAN_Rocklin_2023_3L1X_indels,0.733,0.647,0.584
+UBR5_HUMAN_Rocklin_2023_1I2T_indels,0.893,0.82,0.869
+VG08_BPP22_Rocklin_2023_2GP8_indels,0.851,0.824,0.78
+VILI_CHICK_Rocklin_2023_1YU5_indels,0.932,0.899,0.91
+VRPI_BPT7_Rocklin_2023_2WNM_indels,0.873,0.725,0.77
+YNZC_BACSU_Rocklin_2023_2JVD_indels,0.878,0.759,0.839