{
  "name": "Hugging Face LLM Comparisons",
  "schema_version": "latest",
  "metadata": {
    "parameters": {
      "CoLA_ex_prompt": "Is the sentence grammatical or ungrammatical?\n\n\"This building is than that one.\"",
      "SST_2_ex_prompt": "Is the movie review positive, negative, or neutral?\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"",
      "WNLI_ex_prompt": "Sentence B replaces sentence A's ambiguous pronoun with one of the nouns - is this the correct noun?\n\n\"A) Lily spoke to Donna, breaking her concentration.\nB) Lily spoke to Donna, breaking Lily's concentration.\""
    },
    "models": {},
    "default_model": null,
    "model_parsers": null
  },
  "description": "**In this notebook, we compare the individual performance of HF hosted LLMs () on a few example questions from the GLUE benchmarks (https://gluebenchmark.com/tasks).**\n\n**Example questions taken from \"What is the GLUE Benchmark\" medium post - https://angelina-yang.medium.com/what-is-the-glue-benchmark-for-nlu-systems-61127b3cab3f**\n\n---\n\n| General Language Understanding Evaluation (GLUE) Tasks      | Example Question |\n| ----------- | ----------- |\n| Corpus of Linguistic Acceptability (CoLA)     | Is the sentence grammatical or ungrammatical? \"This building is than that one.\"       |\n| Stanford Sentiment Treebank (SST)   | Is the movie review positive, negative, or neutral? \"The movie is funny, smart, visually inventive, and most of all, alive.\"  |\n| Winograd NLI (WNLI) | Sentence B replaces sentence A's ambiguous pronoun with one of the nouns - is this the correct noun? \"A) Lily spoke to Donna, breaking her concentration. B) Lily spoke to Donna, breaking Lily's concentration.\" |",
  "prompts": [
    {
      "name": "mistral_7b_instruct_v0.1",
      "input": "Is the movie review positive, negative, or neutral?\n\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"",
      "metadata": {
        "model": {
          "name": "Text Generation",
          "settings": {
            "model": "mistralai/Mistral-7B-Instruct-v0.1"
          }
        },
        "tags": null,
        "parameters": {}
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 0,
          "data": "\n\nThe movie review is positive.</s>",
          "mime_type": null,
          "metadata": {}
        }
      ]
    },
    {
      "name": "google_flan_t5_sm",
      "input": "Is the movie review positive, negative, or neutral?\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\"",
      "metadata": {
        "model": {
          "name": "Conversational",
          "settings": {
            "model": "google/flan-t5-small",
            "max_new_tokens": 250,
            "stream": false
          }
        },
        "tags": null,
        "parameters": {}
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 0,
          "data": "positive",
          "mime_type": null,
          "metadata": {
            "raw_response": {
              "generated_text": "positive",
              "conversation": {
                "generated_responses": [
                  "positive"
                ],
                "past_user_inputs": [
                  "Is the movie review positive, negative, or neutral?\n\n\"The movie is funny, smart, visually inventive, and most of all, alive.\""
                ]
              },
              "warnings": [
                "\nNo chat template is defined for this tokenizer - using a default chat template that implements the ChatML format (without BOS/EOS tokens!). If the default is not appropriate for your model, please set `tokenizer.chat_template` to an appropriate template. See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n"
              ]
            }
          }
        }
      ]
    },
    {
      "name": "tinyllama-1_1B",
      "input": "<|system|>\nYou are to answer the following question by the user</s>\n<|user|>\n{{SST_2_ex_prompt}}</s>\n<|assistant|>",
      "metadata": {
        "model": {
          "name": "Conversational",
          "settings": {
            "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
          }
        },
        "tags": null,
        "parameters": {}
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 0,
          "data": "The movie review is positive.",
          "mime_type": null,
          "metadata": {
            "raw_response": {
              "generated_text": "The movie review is positive.",
              "conversation": {
                "generated_responses": [
                  "The movie review is positive."
                ],
                "past_user_inputs": [
                  "<|system|>\nYou are to answer the following question by the user</s>\n<|user|>\nIs the movie review positive, negative, or neutral?\n\n&quot;The movie is funny, smart, visually inventive, and most of all, alive.&quot;</s>\n<|assistant|>"
                ]
              }
            }
          }
        }
      ]
    }
  ]
}