Text edits and added heldout sequences

about.py

@@ -1,4 +1,4 @@
-ABOUT_TEXT = """
+ABOUT_INTRO = """
 ## About this challenge
 
 We're inviting the ML/bio community to predict developability properties for 244 antibodies from the [GDPa1 dataset](https://huggingface.co/datasets/ginkgo-datapoints/GDPa1).
@@ -8,12 +8,26 @@ We're inviting the ML/bio community to predict developability properties for 244
 Antibodies have to be manufacturable, stable in high concentrations, and have low off-target effects.
 Properties such as these can often hinder the progression of an antibody to the clinic, and are collectively referred to as 'developability'.
 Here we show 5 of these properties and invite the community to submit and develop better predictors, which will be tested out on a heldout private set to assess model generalization.
+"""
+
+ABOUT_TEXT = """
+
+**How to participate?**
+
+There are two tracks to the competition:
+- Track 1: If you already have a developability model, you can submit your predictions for the GDPa1 dataset.
+- Track 2: If you don't have a model, train one using cross-validaiton on the GDPa1 dataset and submit your predictions under the "Cross-validation" option. 
+This will provide you with a more accurate estimate of your model's performance on the private test set.
+
+Finally, submit your predictions on the heldout private test set. This will not appear on the leaderboard, and will be used to determine the winners at the close of the competition. 
+There may be some points during the competition where we will release current results on the private test set.
 
 **How to submit?**
 
-1. Download the [GDPa1 dataset](https://huggingface.co/datasets/ginkgo-datapoints/GDPa1)
-2. Make predictions for all the antibody sequences for your property of interest. 
-3. Submit a CSV file containing the `"antibody_name"` column and a column from GDPa1 matching the property name you are predicting (e.g. `"antibody_name,Titer"` if you are predicting Titer). 
+1. Create a Hugging Face account if you don't have one yet (this is used to track unique submissions).
+2. Download the [GDPa1 dataset](https://huggingface.co/datasets/ginkgo-datapoints/GDPa1)
+3. Make predictions for all the antibody sequences for your property of interest. 
+4. Submit a CSV file containing the `"antibody_name"` column and a column from GDPa1 matching the property name you are predicting (e.g. `"antibody_name,Titer"` if you are predicting Titer). 
 There is an example submission file on the "✉️ Submit" tab.
 
 For the cross-validation metrics (if training only on the GDPa1 dataset), use the `"hierarchical_cluster_IgG_isotype_stratified_fold"` column to split the dataset into folds and make predictions for each of the folds. 
@@ -30,13 +44,10 @@ For the heldout private set, we will calculate these results privately at the en
 
 We'd like to add some more existing models to the leaderboard. Some examples of models we'd like to add:
 - ESM embeddings + ridge regression
-- Absolute folding stability models
+- Absolute folding stability models (for Thermostability)
 - AbLEF
 
 If you would like to collaborate with others, start a discussion on the "Community" tab at the top of this page.
-
-### FAQs
-
 """
 # Note(Lood): Let's track these FAQs in the main Google Doc and have that remain the source of truth.
 FAQS = {

app.py

@@ -12,7 +12,7 @@ from constants import (
     EXAMPLE_FILE_DICT,
     LEADERBOARD_DISPLAY_COLUMNS,
 )
-from about import ABOUT_TEXT, FAQS
+from about import ABOUT_INTRO, ABOUT_TEXT, FAQS
 from submit import make_submission
 
 def format_leaderboard_table(df_results: pd.DataFrame, assay: str | None = None):
@@ -80,7 +80,10 @@ with gr.Blocks() as demo:
                 show_download_button=False,
                 width="50vw",  # 50% of the "viewport width"
             )
+            gr.Markdown(ABOUT_INTRO)
+            gr.Image(value="./assets/prediction_explainer.png", show_label=False, show_download_button=False, width="50vw")
             gr.Markdown(ABOUT_TEXT)
+            gr.Markdown("### FAQs")
             for i, (question, answer) in enumerate(FAQS.items()):
                 # Would love to make questions bold but accordion doesn't support it
                 question = f"{i+1}. {question}"
@@ -215,6 +218,7 @@ with gr.Blocks() as demo:
         """
         <div style="text-align: center; font-size: 14px; color: gray; margin-top: 2em;">
         📬 For questions or feedback, contact <a href="mailto:[email protected]">[email protected]</a> or visit the Community tab at the top of this page.
+        Visit the <a href="https://datapoints.ginkgo.bio/ai-competitions/2025-abdev-competition">Competition Registration page</a> to sign up for updates and to register a team.
         </div>
         """,
         elem_id="contact-footer",

constants.py

@@ -41,12 +41,14 @@ CV_COLUMN = "hierarchical_cluster_IgG_isotype_stratified_fold"
 EXAMPLE_FILE_DICT = {
     "GDPa1": "data/example-predictions.csv",
     "GDPa1_cross_validation": "data/example-predictions-cv.csv",
+    "heldout_test": "data/example-predictions-heldout.csv",
 }
 ANTIBODY_NAMES_DICT = {
     "GDPa1": pd.read_csv(EXAMPLE_FILE_DICT["GDPa1"])["antibody_name"].tolist(),
     "GDPa1_cross_validation": pd.read_csv(EXAMPLE_FILE_DICT["GDPa1_cross_validation"])[
         "antibody_name"
     ].tolist(),
+    "heldout_test": pd.read_csv(EXAMPLE_FILE_DICT["heldout_test"])["antibody_name"].tolist(),   # TODO add a test for this validation
 }
 
 # Huggingface API

data/metrics_all.csv

@@ -1,26 +0,0 @@
-assay,model,spearman,spearman_cross_val,top_10_recall,top_10_recall_cross_val
-HIC,Aggrescan3D - aggrescan_average_score,0.422834774225429,,0.3333333333333333,
-AC-SINS_pH7.4,TAP - linear regression,0.4019194824087021,0.3401456689218918,0.375,0.2799999999999999
-HIC,TAP - linear regression,0.3622075317102941,0.2222991438172065,0.4166666666666667,0.43
-AC-SINS_pH7.4,TAP - PNC,0.358762795727933,,0.2916666666666667,
-HIC,Aggrescan3D - aggrescan_90_score,0.3585224061081473,,0.2083333333333333,
-PR_CHO,Saprot_VH - solubility_probability,0.3365516014806938,,0.0833333333333333,
-AC-SINS_pH7.4,TAP - SFvCSP,0.3203773185543964,,0.2083333333333333,
-HIC,Aggrescan3D - aggrescan_max_score,0.3044160918625593,,0.2083333333333333,
-PR_CHO,TAP - linear regression,0.260631929274264,0.1560705020744792,0.8333333333333334,0.47
-HIC,TAP - SFvCSP,0.2450651623577951,,0.2083333333333333,
-PR_CHO,TAP - SFvCSP,0.2381972244142228,,0.0,
-Tm2,Saprot_VH - stability_score,0.1924791603648384,,0.1666666666666666,
-HIC,TAP - CDR Length,0.1923458958277369,,0.0833333333333333,
-Titer,AntiFold,0.1878766623808878,,0.0833333333333333,
-HIC,DeepViscosity,0.18059398754127,,0.0416666666666666,
-AC-SINS_pH7.4,TAP - PPC,0.1691412287169806,,0.0833333333333333,
-Titer,TAP - linear regression,0.1682403605307924,0.1129210260701206,0.3333333333333333,0.38
-PR_CHO,TAP - PNC,0.151234196032203,,0.0416666666666666,
-AC-SINS_pH7.4,TAP - CDR Length,0.1501689804134715,,0.0,
-Titer,TAP - PPC,0.1423756688786398,,0.0833333333333333,
-PR_CHO,Aggrescan3D - aggrescan_max_score,0.1406309504998865,,0.0833333333333333,
-Tm2,AntiFold,0.1218057192943458,,0.125,
-HIC,hic_model_name,0.1144051722170351,0.1511895582680471,0.1666666666666666,0.05
-Tm2,TAP - linear regression,0.0844935706523633,-0.1153965363405958,0.6666666666666666,0.64
-HIC,Aggrescan3D - aggrescan_cdrh3_average_score,0.0747719620306879,,0.25,

data/metrics_all_including_low_spearman.csv

@@ -1,157 +0,0 @@
-feature,assay,spearman,model,spearman_abs
-SFvCSP - tap,HAC,0.6788395883949475,tap,0.6788395883949475
-PPC - tap,HAC,0.4820800038128454,tap,0.4820800038128454
-aggrescan_average_score,HIC,0.3717330854283424,Aggrescan3D,0.3717330854283424
-SFvCSP - tap,PR_Ova,0.3706710402488808,tap,0.3706710402488808
-Viscosity - deep-viscosity,HAC,-0.3683390278828955,DeepViscosity,0.3683390278828955
-aggrescan_average_score,SMAC,0.3643128778304258,Aggrescan3D,0.3643128778304258
-PNC - tap,AC-SINS_pH7.4,-0.358762795727933,tap,0.358762795727933
-aggrescan_90_score,HIC,0.3572823382390205,Aggrescan3D,0.3572823382390205
-solubility_probability - saprot,PR_CHO,0.3365516014806938,saprot,0.3365516014806938
-PNC - tap,HAC,-0.3206575607459202,tap,0.3206575607459202
-SFvCSP - tap,AC-SINS_pH7.4,0.3203773185543964,tap,0.3203773185543964
-PNC - tap,PR_Ova,-0.2932051856162034,tap,0.2932051856162034
-aggrescan_90_score,SMAC,0.2852401926417188,Aggrescan3D,0.2852401926417188
-aggrescan_max_score,SMAC,0.2720151775798609,Aggrescan3D,0.2720151775798609
-aggrescan_max_score,HIC,0.2579163714894611,Aggrescan3D,0.2579163714894611
-stability_score - saprot,HAC,0.2482190338151043,saprot,0.2482190338151043
-SFvCSP - tap,HIC,-0.2450651623577951,tap,0.2450651623577951
-aggrescan_max_score,PR_CHO,-0.24241781163854,Aggrescan3D,0.24241781163854
-SFvCSP - tap,PR_CHO,0.2381972244142228,tap,0.2381972244142228
-PNC - tap,AC-SINS_pH6.0,-0.2185768523842327,tap,0.2185768523842327
-PPC - tap,PR_Ova,0.205520359496246,tap,0.205520359496246
-aggrescan_max_score,PR_Ova,-0.1983067762197625,Aggrescan3D,0.1983067762197625
-stability_score - saprot,Tm2,-0.1924791603648384,saprot,0.1924791603648384
-CDR Length - tap,HIC,0.1923458958277369,tap,0.1923458958277369
-solubility_probability - saprot,PR_Ova,0.1859602281879885,saprot,0.1859602281879885
-PPC - tap,HIC,-0.1839687842860175,tap,0.1839687842860175
-aggrescan_cdrh3_average_score,Titer,0.1814699248452822,Aggrescan3D,0.1814699248452822
-Viscosity - deep-viscosity,HIC,0.18059398754127,DeepViscosity,0.18059398754127
-PPC - tap,SEC %Monomer,0.179898358348615,tap,0.179898358348615
-Viscosity - deep-viscosity,PR_Ova,-0.1792011299598071,DeepViscosity,0.1792011299598071
-aggrescan_cdrh3_average_score,Purity,-0.1774879106041495,Aggrescan3D,0.1774879106041495
-CDR Length - tap,SMAC,0.1753482083680697,tap,0.1753482083680697
-PPC - tap,Purity,-0.1743385504957838,tap,0.1743385504957838
-PPC - tap,Tm1,0.1717910377244919,tap,0.1717910377244919
-PPC - tap,AC-SINS_pH7.4,0.1691412287169806,tap,0.1691412287169806
-PNC - tap,SMAC,-0.1647616991330237,tap,0.1647616991330237
-SFvCSP - tap,AC-SINS_pH6.0,0.1631061026200202,tap,0.1631061026200202
-aggrescan_90_score,PR_Ova,-0.1630725493267146,Aggrescan3D,0.1630725493267146
-aggrescan_cdrh3_average_score,HAC,-0.1621653601888134,Aggrescan3D,0.1621653601888134
-aggrescan_90_score,PR_CHO,-0.1602601604994121,Aggrescan3D,0.1602601604994121
-PNC - tap,PR_CHO,-0.151234196032203,tap,0.151234196032203
-CDR Length - tap,AC-SINS_pH7.4,-0.1501689804134715,tap,0.1501689804134715
-stability_score - saprot,PR_CHO,0.1482133956646727,saprot,0.1482133956646727
-PPC - tap,AC-SINS_pH6.0,0.1448272149205899,tap,0.1448272149205899
-SFvCSP - tap,Tm1,0.1426983792732488,tap,0.1426983792732488
-PPC - tap,Titer,0.1423756688786398,tap,0.1423756688786398
-PNC - tap,SEC %Monomer,-0.1345634135302046,tap,0.1345634135302046
-aggrescan_average_score,HAC,-0.1311020701331903,Aggrescan3D,0.1311020701331903
-aggrescan_cdrh3_average_score,Tm2,-0.1295581308418123,Aggrescan3D,0.1295581308418123
-PSH - tap,HAC,0.1269358458430945,tap,0.1269358458430945
-aggrescan_cdrh3_average_score,AC-SINS_pH6.0,0.1241248052844776,Aggrescan3D,0.1241248052844776
-aggrescan_average_score,PR_Ova,-0.1225803596842919,Aggrescan3D,0.1225803596842919
-solubility_probability - saprot,Purity,0.120581879119953,saprot,0.120581879119953
-aggrescan_90_score,AC-SINS_pH6.0,-0.1156038989665139,Aggrescan3D,0.1156038989665139
-Viscosity - deep-viscosity,PR_CHO,-0.1125214482263828,DeepViscosity,0.1125214482263828
-aggrescan_cdrh3_average_score,HIC,0.1109333602778311,Aggrescan3D,0.1109333602778311
-PNC - tap,Tm1,-0.1094376940826625,tap,0.1094376940826625
-PNC - tap,Titer,-0.1043862069630446,tap,0.1043862069630446
-aggrescan_90_score,HAC,-0.1029367717205433,Aggrescan3D,0.1029367717205433
-PPC - tap,Tonset,0.1024480318260903,tap,0.1024480318260903
-aggrescan_average_score,Tm1,0.0984625979382824,Aggrescan3D,0.0984625979382824
-SFvCSP - tap,Purity,-0.0979379393217746,tap,0.0979379393217746
-solubility_probability - saprot,Tm2,-0.0978456145482691,saprot,0.0978456145482691
-SFvCSP - tap,SEC %Monomer,0.096650605592591,tap,0.096650605592591
-Viscosity - deep-viscosity,Tm2,0.093745084007507,DeepViscosity,0.093745084007507
-PNC - tap,Tonset,-0.0925140513893314,tap,0.0925140513893314
-aggrescan_90_score,Titer,-0.0888890152050186,Aggrescan3D,0.0888890152050186
-PPC - tap,PR_CHO,0.0885325199884014,tap,0.0885325199884014
-Viscosity - deep-viscosity,SMAC,0.0883647279655625,DeepViscosity,0.0883647279655625
-PPC - tap,SMAC,-0.0881450436285762,tap,0.0881450436285762
-SFvCSP - tap,Tonset,0.0869080829604942,tap,0.0869080829604942
-aggrescan_average_score,Purity,-0.0855300736249475,Aggrescan3D,0.0855300736249475
-PSH - tap,PR_Ova,0.0851843571887952,tap,0.0851843571887952
-solubility_probability - saprot,HAC,-0.0849192244020849,saprot,0.0849192244020849
-stability_score - saprot,PR_Ova,0.0844385315275266,saprot,0.0844385315275266
-aggrescan_average_score,PR_CHO,-0.082335263587056,Aggrescan3D,0.082335263587056
-CDR Length - tap,HAC,-0.0807572482895502,tap,0.0807572482895502
-Viscosity - deep-viscosity,AC-SINS_pH7.4,-0.0803879126171943,DeepViscosity,0.0803879126171943
-SFvCSP - tap,Titer,0.0797623802296775,tap,0.0797623802296775
-aggrescan_average_score,AC-SINS_pH7.4,0.078198839245547,Aggrescan3D,0.078198839245547
-aggrescan_max_score,Tonset,-0.0758666681469678,Aggrescan3D,0.0758666681469678
-stability_score - saprot,SMAC,-0.0752343621503858,saprot,0.0752343621503858
-stability_score - saprot,Tonset,-0.0738992116600914,saprot,0.0738992116600914
-stability_score - saprot,SEC %Monomer,-0.0733865038928551,saprot,0.0733865038928551
-CDR Length - tap,Tm2,-0.0729274519946903,tap,0.0729274519946903
-CDR Length - tap,SEC %Monomer,-0.0720782456119583,tap,0.0720782456119583
-stability_score - saprot,AC-SINS_pH7.4,0.0718305679545067,saprot,0.0718305679545067
-solubility_probability - saprot,Tonset,-0.0707794898935402,saprot,0.0707794898935402
-CDR Length - tap,Tonset,-0.0697571003932752,tap,0.0697571003932752
-solubility_probability - saprot,AC-SINS_pH6.0,0.0669921910968805,saprot,0.0669921910968805
-PSH - tap,SMAC,-0.0668504875953866,tap,0.0668504875953866
-CDR Length - tap,AC-SINS_pH6.0,-0.0662557372283483,tap,0.0662557372283483
-aggrescan_90_score,Purity,-0.0642609945737365,Aggrescan3D,0.0642609945737365
-CDR Length - tap,Purity,0.0640138427858363,tap,0.0640138427858363
-aggrescan_cdrh3_average_score,PR_CHO,-0.0639580648777149,Aggrescan3D,0.0639580648777149
-PSH - tap,Titer,-0.0638862853735791,tap,0.0638862853735791
-Viscosity - deep-viscosity,Titer,-0.0636053342823435,DeepViscosity,0.0636053342823435
-stability_score - saprot,HIC,-0.0625972585374469,saprot,0.0625972585374469
-stability_score - saprot,Titer,0.0607177541361877,saprot,0.0607177541361877
-aggrescan_average_score,Tonset,0.0532502341076833,Aggrescan3D,0.0532502341076833
-CDR Length - tap,PR_CHO,-0.0524556823713343,tap,0.0524556823713343
-aggrescan_90_score,AC-SINS_pH7.4,-0.0509589937861982,Aggrescan3D,0.0509589937861982
-PSH - tap,SEC %Monomer,-0.0487785361924088,tap,0.0487785361924088
-PNC - tap,Tm2,-0.0482259337861972,tap,0.0482259337861972
-SFvCSP - tap,SMAC,-0.0479604170867368,tap,0.0479604170867368
-CDR Length - tap,Tm1,0.0470910111346684,tap,0.0470910111346684
-PSH - tap,HIC,-0.0464732086283245,tap,0.0464732086283245
-aggrescan_cdrh3_average_score,SMAC,0.0462084030163925,Aggrescan3D,0.0462084030163925
-stability_score - saprot,Tm1,-0.0461587451740238,saprot,0.0461587451740238
-aggrescan_max_score,AC-SINS_pH6.0,-0.0422041355140005,Aggrescan3D,0.0422041355140005
-Viscosity - deep-viscosity,Tm1,0.0418758583153736,DeepViscosity,0.0418758583153736
-CDR Length - tap,Titer,-0.0415639454664855,tap,0.0415639454664855
-Viscosity - deep-viscosity,AC-SINS_pH6.0,-0.0391995375434282,DeepViscosity,0.0391995375434282
-aggrescan_max_score,SEC %Monomer,0.0384606168278266,Aggrescan3D,0.0384606168278266
-aggrescan_max_score,Purity,0.0377965473734412,Aggrescan3D,0.0377965473734412
-aggrescan_cdrh3_average_score,Tonset,-0.0371645511788307,Aggrescan3D,0.0371645511788307
-stability_score - saprot,AC-SINS_pH6.0,0.0363459090170338,saprot,0.0363459090170338
-PSH - tap,AC-SINS_pH7.4,-0.0360881689253704,tap,0.0360881689253704
-solubility_probability - saprot,Tm1,-0.0359790946528558,saprot,0.0359790946528558
-aggrescan_average_score,SEC %Monomer,0.0329005393127016,Aggrescan3D,0.0329005393127016
-solubility_probability - saprot,AC-SINS_pH7.4,0.0320374764320225,saprot,0.0320374764320225
-stability_score - saprot,Purity,0.0319067204658559,saprot,0.0319067204658559
-solubility_probability - saprot,HIC,0.0318219345388521,saprot,0.0318219345388521
-PSH - tap,Tm2,-0.0317350623491362,tap,0.0317350623491362
-aggrescan_average_score,Tm2,-0.0312770789755912,Aggrescan3D,0.0312770789755912
-aggrescan_cdrh3_average_score,AC-SINS_pH7.4,0.0305591682601827,Aggrescan3D,0.0305591682601827
-aggrescan_max_score,Tm1,-0.0301284126829971,Aggrescan3D,0.0301284126829971
-aggrescan_cdrh3_average_score,Tm1,-0.0293641493096082,Aggrescan3D,0.0293641493096082
-aggrescan_90_score,Tm1,0.0279545108349369,Aggrescan3D,0.0279545108349369
-aggrescan_average_score,AC-SINS_pH6.0,0.0271292409530908,Aggrescan3D,0.0271292409530908
-PSH - tap,Tonset,0.0258012823141481,tap,0.0258012823141481
-solubility_probability - saprot,SEC %Monomer,-0.0255220000167489,saprot,0.0255220000167489
-aggrescan_cdrh3_average_score,SEC %Monomer,-0.0242523887163972,Aggrescan3D,0.0242523887163972
-aggrescan_max_score,HAC,-0.0238463005336014,Aggrescan3D,0.0238463005336014
-Viscosity - deep-viscosity,SEC %Monomer,-0.0226036392916792,DeepViscosity,0.0226036392916792
-aggrescan_max_score,Titer,-0.0218947825112391,Aggrescan3D,0.0218947825112391
-PSH - tap,PR_CHO,0.0209381900169799,tap,0.0209381900169799
-aggrescan_max_score,Tm2,-0.0199308718790671,Aggrescan3D,0.0199308718790671
-PSH - tap,Purity,-0.0198292183035963,tap,0.0198292183035963
-CDR Length - tap,PR_Ova,-0.0191507855521124,tap,0.0191507855521124
-SFvCSP - tap,Tm2,-0.0179541460075494,tap,0.0179541460075494
-aggrescan_max_score,AC-SINS_pH7.4,0.0177424118352705,Aggrescan3D,0.0177424118352705
-Viscosity - deep-viscosity,Purity,-0.0164122333153846,DeepViscosity,0.0164122333153846
-PSH - tap,Tm1,0.0149070260126188,tap,0.0149070260126188
-PNC - tap,HIC,0.0102627218889135,tap,0.0102627218889135
-PPC - tap,Tm2,-0.0098872681569066,tap,0.0098872681569066
-PNC - tap,Purity,0.0096311210026764,tap,0.0096311210026764
-aggrescan_90_score,SEC %Monomer,-0.0069533060784548,Aggrescan3D,0.0069533060784548
-solubility_probability - saprot,Titer,-0.0068735643986468,saprot,0.0068735643986468
-aggrescan_cdrh3_average_score,PR_Ova,-0.0057045278689166,Aggrescan3D,0.0057045278689166
-Viscosity - deep-viscosity,Tonset,-0.0052886682383666,DeepViscosity,0.0052886682383666
-aggrescan_90_score,Tm2,0.0014898396535596,Aggrescan3D,0.0014898396535596
-aggrescan_90_score,Tonset,-0.0012258917323618,Aggrescan3D,0.0012258917323618
-solubility_probability - saprot,SMAC,-0.0008774163836285,saprot,0.0008774163836285
-aggrescan_average_score,Titer,-0.0003490541872287,Aggrescan3D,0.0003490541872287
-PSH - tap,AC-SINS_pH6.0,-6.566404038227427e-05,tap,6.566404038227427e-05

utils.py

@@ -1,4 +1,3 @@
-from datetime import datetime, timezone, timedelta
 import pandas as pd
 from datasets import load_dataset
 import gradio as gr
@@ -13,9 +12,10 @@ def show_output_box(message):
 
 
 def fetch_hf_results():
-    # Print current time in EST
-    EST = timezone(timedelta(hours=-4))
-    print(f"tmp: Fetching results from HF at {datetime.now(EST)}") 
+    # For debugging
+    # # Print current time in EST
+    # EST = timezone(timedelta(hours=-4))
+    # print(f"tmp: Fetching results from HF at {datetime.now(EST)}") 
     # Should cache by default if not using force_redownload
     df = load_dataset(
         RESULTS_REPO, data_files="auto_submissions/metrics_all.csv",