README.md

metadata

base_model:
  - nvidia/Llama-3.1-Nemotron-70B-Instruct-HF
  - meta-llama/Llama-3.1-70B-Instruct
  - ValiantLabs/Llama3.1-70B-ShiningValiant2
language:
  - en
library_name: transformers
license: llama3.1
tags:
  - llama
  - llama3.1
  - llama3
  - meta
  - 70b
  - science
  - physics
  - biology
  - chemistry
  - compsci
  - computer-science
  - engineering
  - logic
  - rationality
  - advanced
  - expert
  - technical
  - conversational
  - chat
  - instruct
  - mergekit
  - merge
pipeline_tag: text-generation
model_type: llama
model-index:
  - name: sequelbox/Llama3.1-70B-PlumChat
    results:
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: Winogrande (5-Shot)
          type: Winogrande
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 85
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: ARC Challenge (25-Shot)
          type: arc-challenge
          args:
            num_few_shot: 25
        metrics:
          - type: acc_norm
            value: 67.41
            name: normalized accuracy
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU College Biology (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 93.75
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU High School Biology (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 91.94
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU Conceptual Physics (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 82.13
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU College Physics (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 60.78
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU High School Physics (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 62.25
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU College Chemistry (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 56
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU High School Chemistry (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 73.4
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU Astronomy (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 89.47
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU College Computer Science (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 64
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU High School Computer Science (5-Shot)
          type: MMLU
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 90
            name: acc
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: IFEval (0-Shot)
          type: HuggingFaceH4/ifeval
          args:
            num_few_shot: 0
        metrics:
          - type: inst_level_strict_acc and prompt_level_strict_acc
            value: 56.16
            name: strict accuracy
        source:
          url: >-
            https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=sequelbox/Llama3.1-8B-PlumChat
          name: Open LLM Leaderboard
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: BBH (3-Shot)
          type: BBH
          args:
            num_few_shot: 3
        metrics:
          - type: acc_norm
            value: 52.81
            name: normalized accuracy
        source:
          url: >-
            https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=sequelbox/Llama3.1-8B-PlumChat
          name: Open LLM Leaderboard
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MATH Lvl 5 (4-Shot)
          type: hendrycks/competition_math
          args:
            num_few_shot: 4
        metrics:
          - type: exact_match
            value: 29.98
            name: exact match
        source:
          url: >-
            https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=sequelbox/Llama3.1-8B-PlumChat
          name: Open LLM Leaderboard
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: GPQA (0-shot)
          type: Idavidrein/gpqa
          args:
            num_few_shot: 0
        metrics:
          - type: acc_norm
            value: 18.79
            name: acc_norm
        source:
          url: >-
            https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=sequelbox/Llama3.1-8B-PlumChat
          name: Open LLM Leaderboard
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MuSR (0-shot)
          type: TAUR-Lab/MuSR
          args:
            num_few_shot: 0
        metrics:
          - type: acc_norm
            value: 20.14
            name: acc_norm
        source:
          url: >-
            https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=sequelbox/Llama3.1-8B-PlumChat
          name: Open LLM Leaderboard
      - task:
          type: text-generation
          name: Text Generation
        dataset:
          name: MMLU-PRO (5-shot)
          type: TIGER-Lab/MMLU-Pro
          config: main
          split: test
          args:
            num_few_shot: 5
        metrics:
          - type: acc
            value: 46.26
            name: accuracy
        source:
          url: >-
            https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=sequelbox/Llama3.1-8B-PlumChat
          name: Open LLM Leaderboard

PlumChat 70b

This is a merge of pre-trained language models created using mergekit.

Merge Details

Shining Valiant 2 + Nemotron for high quality general chat, science-instruct, and complex query performance.

Merge Method

This model was merged using the della merge method using meta-llama/Llama-3.1-70B-Instruct as a base.

Models Merged

The following models were included in the merge:

• nvidia/Llama-3.1-Nemotron-70B-Instruct-HF
• ValiantLabs/Llama3.1-70B-ShiningValiant2

Configuration

The following YAML configuration was used to produce this model:

merge_method: della
dtype: bfloat16
parameters:
  normalize: true
models:
  - model: nvidia/Llama-3.1-Nemotron-70B-Instruct-HF
    parameters:
      density: 0.5
      weight: 0.3
  - model: ValiantLabs/Llama3.1-70B-ShiningValiant2
    parameters:
      density: 0.5
      weight: 0.25
base_model: meta-llama/Llama-3.1-70B-Instruct