Fixed app v2

app.py

@@ -1,20 +1,24 @@
-from transformers import AutoTokenizer, pipeline
+from transformers import AutoTokenizer
+from optimum.intel import INCModelForSeq2SeqLM
 from optimum.intel.openvino import OVModelForCausalLM
 import gradio as gr
 import json
 
 # Load OpenVINO GPT-J model for causal language modeling
-model_id = "OpenVINO/gpt-j-6b-int4-ov"
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-model = OVModelForCausalLM.from_pretrained(model_id)
+causal_model_id = "OpenVINO/gpt-j-6b-int4-ov"
+tokenizer = AutoTokenizer.from_pretrained(causal_model_id)
+causal_model = OVModelForCausalLM.from_pretrained(causal_model_id)
 
-# Load a quantized summarization model
-summarizer_model_id = "OpenVINO/distilbart-cnn-12-6-int8-ov"  # Example of a quantized summarization model
-summarizer = pipeline("summarization", model=summarizer_model_id)
+# Load the Intel quantized summarization model
+summarizer_model_id = "Intel/distilbart-cnn-12-6-int8-dynamic"
+tokenizer_summarizer = AutoTokenizer.from_pretrained(summarizer_model_id)
+int8_model = INCModelForSeq2SeqLM.from_pretrained(summarizer_model_id)
 
 def summarize_text(text, max_length=100):
-    summary = summarizer(text, max_length=max_length, min_length=25, do_sample=False)
-    return summary[0]["summary_text"]
+    inputs = tokenizer_summarizer(text, return_tensors="pt", max_length=512, truncation=True)
+    summary_ids = int8_model.generate(inputs.input_ids, max_length=max_length, min_length=25, do_sample=False)
+    summary = tokenizer_summarizer.decode(summary_ids[0], skip_special_tokens=True)
+    return summary
 
 def match_cv_to_jobs(cv_text, job_descriptions_text):
     debug_info = "Debug Info:\n"
@@ -31,14 +35,14 @@ def match_cv_to_jobs(cv_text, job_descriptions_text):
     # Create a prompt to compare the summarized CV with the summarized job descriptions
     prompt = (
         f"Compare the following job descriptions with this resume. Job Descriptions: {summarized_descriptions}. "
-        f"Resume: {summarized_cv}. Provide a match score ONLY out of 100 "
+        f"Resume: {summarized_cv}. Provide a match score and a brief analysis."
     )
     debug_info += f"\nGenerated Prompt: {prompt}\n"
     
-    # Generate response from the model
+    # Generate response from the causal model
     inputs = tokenizer(prompt, return_tensors="pt")
     try:
-        outputs = model.generate(**inputs, max_length=200)
+        outputs = causal_model.generate(**inputs, max_length=200)
         response_content = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
         debug_info += f"Model Response: {response_content}\n"