| { | |
| "name": "Hugging Face LLM Comparisons", | |
| "schema_version": "latest", | |
| "metadata": { | |
| "parameters": { | |
| "CoLA_ex_prompt": "Is the sentence grammatical or ungrammatical?\n\n\"This building is than that one.\"", | |
| "SST_2_ex_prompt": "Is the movie review positive, negative, or neutral?\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"", | |
| "WNLI_ex_prompt": "Sentence B replaces sentence A's ambiguous pronoun with one of the nouns - is this the correct noun?\n\n\"A) Lily spoke to Donna, breaking her concentration.\nB) Lily spoke to Donna, breaking Lily's concentration.\"" | |
| }, | |
| "models": {}, | |
| "default_model": null, | |
| "model_parsers": null | |
| }, | |
| "description": "**In this notebook, we compare the individual performance of HF hosted LLMs () on a few example questions from the GLUE benchmarks (https://gluebenchmark.com/tasks).**\n\n**Example questions taken from \"What is the GLUE Benchmark\" medium post - https://angelina-yang.medium.com/what-is-the-glue-benchmark-for-nlu-systems-61127b3cab3f**\n\n---\n\n| General Language Understanding Evaluation (GLUE) Tasks | Example Question |\n| ----------- | ----------- |\n| Corpus of Linguistic Acceptability (CoLA) | Is the sentence grammatical or ungrammatical? \"This building is than that one.\" |\n| Stanford Sentiment Treebank (SST) | Is the movie review positive, negative, or neutral? \"The movie is funny, smart, visually inventive, and most of all, alive.\" |\n| Winograd NLI (WNLI) | Sentence B replaces sentence A's ambiguous pronoun with one of the nouns - is this the correct noun? \"A) Lily spoke to Donna, breaking her concentration. B) Lily spoke to Donna, breaking Lily's concentration.\" |", | |
| "prompts": [ | |
| { | |
| "name": "mistral_7b_instruct_v0.1", | |
| "input": "Is the movie review positive, negative, or neutral?\n\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"", | |
| "metadata": { | |
| "model": { | |
| "name": "Text Generation", | |
| "settings": { | |
| "model": "mistralai/Mistral-7B-Instruct-v0.1" | |
| } | |
| }, | |
| "tags": null, | |
| "parameters": {} | |
| }, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "execution_count": 0, | |
| "data": "\n\nThe movie review is positive.</s>", | |
| "mime_type": null, | |
| "metadata": {} | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "google_flan_t5_sm", | |
| "input": "Is the movie review positive, negative, or neutral?\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"", | |
| "metadata": { | |
| "model": { | |
| "name": "Conversational", | |
| "settings": { | |
| "model": "google/flan-t5-small", | |
| "max_new_tokens": 250, | |
| "stream": false | |
| } | |
| }, | |
| "tags": null, | |
| "parameters": {} | |
| }, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "execution_count": 0, | |
| "data": "positive", | |
| "mime_type": null, | |
| "metadata": { | |
| "raw_response": { | |
| "generated_text": "positive", | |
| "conversation": { | |
| "generated_responses": [ | |
| "positive" | |
| ], | |
| "past_user_inputs": [ | |
| "Is the movie review positive, negative, or neutral?\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"" | |
| ] | |
| }, | |
| "warnings": [ | |
| "\nNo chat template is defined for this tokenizer - using a default chat template that implements the ChatML format (without BOS/EOS tokens!). If the default is not appropriate for your model, please set `tokenizer.chat_template` to an appropriate template. See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n" | |
| ] | |
| } | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "tinyllama-1_1B", | |
| "input": "<|system|>\nYou are to answer the following question by the user</s>\n<|user|>\n{{SST_2_ex_prompt}}</s>\n<|assistant|>", | |
| "metadata": { | |
| "model": { | |
| "name": "Conversational", | |
| "settings": { | |
| "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0" | |
| } | |
| }, | |
| "tags": null, | |
| "parameters": {} | |
| }, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "execution_count": 0, | |
| "data": "The movie review is positive.", | |
| "mime_type": null, | |
| "metadata": { | |
| "raw_response": { | |
| "generated_text": "The movie review is positive.", | |
| "conversation": { | |
| "generated_responses": [ | |
| "The movie review is positive." | |
| ], | |
| "past_user_inputs": [ | |
| "<|system|>\nYou are to answer the following question by the user</s>\n<|user|>\nIs the movie review positive, negative, or neutral?\n\n"The movie is funny, smart, visually inventive, and most of all, alive."</s>\n<|assistant|>" | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| ] | |
| } | |
| ] | |
| } |