cache hf models

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/babelmachine-dev.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,33 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="13">
+            <item index="0" class="java.lang.String" itemvalue="google-cloud-bigquery" />
+            <item index="1" class="java.lang.String" itemvalue="db-dtypes" />
+            <item index="2" class="java.lang.String" itemvalue="pytest" />
+            <item index="3" class="java.lang.String" itemvalue="google-cloud-secret-manager" />
+            <item index="4" class="java.lang.String" itemvalue="redis" />
+            <item index="5" class="java.lang.String" itemvalue="google-cloud-logging" />
+            <item index="6" class="java.lang.String" itemvalue="functions_framework" />
+            <item index="7" class="java.lang.String" itemvalue="google-cloud-compute" />
+            <item index="8" class="java.lang.String" itemvalue="pickle5" />
+            <item index="9" class="java.lang.String" itemvalue="typing_extensions" />
+            <item index="10" class="java.lang.String" itemvalue="label_studio_sdk" />
+            <item index="11" class="java.lang.String" itemvalue="gcsfs" />
+            <item index="12" class="java.lang.String" itemvalue="rq" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredIdentifiers">
+        <list>
+          <option value="mido.*" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/babelmachine-dev.iml" filepath="$PROJECT_DIR$/.idea/babelmachine-dev.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

app.py

@@ -1,4 +1,3 @@
-import os
 import gradio as gr
 
 from spacy import glossary
@@ -8,8 +7,7 @@ from interfaces.sentiment import demo as sentiment_demo
 from interfaces.emotion import demo as emotion_demo
 from interfaces.ner import demo as ner_demo
 from interfaces.ner import download_models as download_spacy_models
-
-os.environ['TRANSFORMERS_CACHE'] = '/data/'
+from utils import download_hf_models
 
 entities = ["CARDINAL", "DATE", "EVENT", "FAC", "GPE", "LANGUAGE", "LAW", "LOC", "MONEY", "NORP", "ORDINAL", "ORG", "PERCENT", "PERSON", "PRODUCT", "QUANTITY", "TIME", "WORK_OF_ART"]
 ent_dict = glossary.GLOSSARY
@@ -35,9 +33,8 @@ with gr.Blocks() as demo:
     )
 
 if __name__ == "__main__":
-    gr.Info("Downloading spacy models")
+    download_hf_models()
     download_spacy_models()
-    gr.Info("Downloaded spacy models")
     demo.launch()
 
 # TODO: add all languages & domains

interfaces/manifesto.py

@@ -39,6 +39,8 @@ def build_huggingface_path(language: str):
     return "poltextlab/xlm-roberta-large-manifesto"
 
 def predict(text, model_id, tokenizer_id):
+    gr.Info(os.listdir("/data/"))
+
     device = torch.device("cpu")
     with m("Loading model"):
         model = AutoModelForSequenceClassification.from_pretrained(model_id, low_cpu_mem_usage=True, device_map="auto", token=HF_TOKEN)

utils.py

@@ -0,0 +1,30 @@
+import os
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+
+"""
+from interfaces.manifesto import languages as languages_manifesto
+from interfaces.manifesto import languages as languages_manifesto
+from interfaces.manifesto import languages as languages_manifesto
+"""
+
+from interfaces.cap import build_huggingface_path as hf_cap_path
+from interfaces.manifesto import build_huggingface_path as hf_manifesto_path
+from interfaces.sentiment import build_huggingface_path as hf_sentiment_path
+from interfaces.emotion import build_huggingface_path as hf_emotion_path
+
+
+os.environ['TRANSFORMERS_CACHE'] = '/data/'
+HF_TOKEN = os.environ["hf_read"]
+
+models = [hf_manifesto_path(""), hf_sentiment_path(""), hf_emotion_path("")]
+tokenizers = ["xlm-roberta-large"]
+
+def download_hf_models():
+    for model in models:
+        model = AutoModelForSequenceClassification.from_pretrained(model_id, low_cpu_mem_usage=True, device_map="auto",
+                                                                   token=HF_TOKEN)
+    del model
+    for tokenizer in tokenizers:
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
+    del tokenizer
+