lm/server_lm/serve_no_stream.py

import os
import os
from pathlib import Path
import csv
import json
import time
import pandas as pd
import requests
from PIL import Image
import time
from io import BytesIO
import json
# Set up the OpenAI API client
api_key = "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"


# Set up the chatGPT model and prompt
model_engine = "text-davinci-003"
import gradio as gr
import time
import argparse
from vllm import LLM, SamplingParams

def call_async_server(prompt):
    json_object = {"prompt": prompt,
                   # "max_tokens": 2048000,
                   "max_tokens": 2048000,
                   "n": 1
                   }
    generate_response = requests.post("http://phlrr3058.guest.corp.microsoft.com:7860/generate", json=json_object)
    end = time.time()
    res_json = json.loads(generate_response.content)
    ASSISTANT = res_json['text'][-1].split("ASSISTANT:")[-1].strip()
    print(ASSISTANT)
    return ASSISTANT

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--model", type=str)  # model path
    parser.add_argument("--n_gpu", type=int, default=1)  # n_gpu
    return parser.parse_args()

def echo(message, history, system_prompt, temperature, max_tokens):
    response = f"System prompt: {system_prompt}\n Message: {message}. \n Temperature: {temperature}. \n Max Tokens: {max_tokens}."
    for i in range(min(len(response), int(max_tokens))):
        time.sleep(0.05)
        yield response[: i+1]



def align_data(data):
    """Given dict with lists, creates aligned strings

    Adapted from Assignment 3 of CS224N

    Args:
        data: (dict) data["x"] = ["I", "love", "you"]
              (dict) data["y"] = ["O", "O", "O"]

    Returns:
        data_aligned: (dict) data_align["x"] = "I love you"
                           data_align["y"] = "O O    O  "

    """
    spacings = [max([len(seq[i]) for seq in data.values()])
                for i in range(len(data[list(data.keys())[0]]))]
    data_aligned = dict()

    # for each entry, create aligned string
    for key, seq in data.items():
        str_aligned = ""
        for token, spacing in zip(seq, spacings):
            str_aligned += token + " " * (spacing - len(token) + 1)

        data_aligned[key] = str_aligned

    return data_aligned

# def get_llm_result(input_data, input_domain):
def get_llm_result(input_sys_prompt_str, input_history_str, prompt_str):
    # data is file path of topic result
    prompt = ""

    def predict(message, history, system_prompt, temperature, max_tokens):
        # instruction = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. "
        instruction = system_prompt
        for human, assistant in history:
            instruction += 'USER: '+ human + ' ASSISTANT: '+ assistant + '</s>'
        instruction += 'USER: '+ message + ' ASSISTANT:'
        # problem = [instruction]
        # stop_tokens = ["Question:", "Question", "USER:", "USER", "ASSISTANT:", "ASSISTANT", "Instruction:", "Instruction", "Response:", "Response"]
        # sampling_params = SamplingParams(temperature=temperature, top_p=1, max_tokens=max_tokens, stop=stop_tokens)
        # completions = llm.generate(problem, sampling_params)
        generate_rext = call_async_server(instruction)
        return generate_rext
        # for output in completions:
        #     prompt = output.prompt
        #     generated_text = output.outputs[0].text
        #     return generated_text
            # for idx in range(len(generated_text)):
            #     yield generated_text[:idx+1]
    try:
        # completion = openai.Completion.create(
        #     engine=model_engine,
        #     prompt=prompt,
        #     max_tokens=3000,
        #     n=1,
        #     stop=None,
        #     temperature=0.5,
        # )
        #
        # response = completion.choices[0].text
        # shorten_response = response.replace("\n", "").strip()
        # len_response = len(shorten_response.split(" "))
        # if len_response >= 3500:
        #     shorten_response = "".join(shorten_response.split(" ")[:3500])
        #     print("X"*10)
        #     print(f"shorten_response is {shorten_response}")
        #     list_shorten = shorten_response.split(" ")
        #     print(list_shorten)
        #     print(f"length is {len(list_shorten)}")
        # title_prompt = f"{shorten_response}，给这个文章写一个头条号风格的标题。增加标题的吸引力，可读性。"
        # title_completion = openai.Completion.create(
        #     engine=model_engine,
        #     prompt=title_prompt,
        #     max_tokens=200,
        #     n=1,
        #     stop=None,
        #     temperature=0.5,
        # )
        # title_response = title_completion.choices[0].text
        # model_path = "/workspaceblobstore/caxu/trained_models/13Bv2_497kcontinueroleplay_dsys_2048_e4_2e_5/checkpoint-75"
        # llm = LLM(model=model_path, tensor_parallel_size=1)
        history = input_history_str
        prompt = prompt_str
        system_prompt = input_sys_prompt_str

        response = predict(prompt, history, system_prompt, 0.5, 3000)

        print(response)
        # if not os.path.isdir(topic_file_path):
        #     print("File folder  not exist")
        # topic_result_file = ""
        # topic_file_name_pattern = "step10_json_filestep9_merge_rewrite_"
        # for filename in os.listdir(topic_file_path):
        #     if filename.startswith(topic_file_name_pattern):
        #         topic_result_file = os.path.join(topic_file_path, filename)
        #
        # data_aligned = dict()
        # output_dir_name = "."
        # output_dir = os.path.join(output_dir_name, "result_topic_file")
        # Path(output_dir).mkdir(parents=True, exist_ok=True)
        # write_file_name = "save_server_" + topic_file_path.split("\\")[-1]
        # write_output_file_path = os.path.join(output_dir, write_file_name)
        #
        # with open(topic_result_file, encoding="utf8") as f:
        #         json_data = json.load(f)
        #         return json_data
        return response, response

    except Exception as ex:
        print("File  not exist")
        raise ex

def get_model_api():
    """Returns lambda function for api"""

    # def model_api(input_title, input_domain):
    def model_api(input_sys_prompt_str, input_history_str, prompt_str):
        """
        Args:
            input_data: submitted to the API, raw string

        Returns:
            output_data: after some transformation, to be
                returned to the API

        """
        # print("X"*10)
        # print(f"input_title is {input_title}")
        # print(f"input_data2 is {input_domain}")
        punc = [",", "?", ".", ":", ";", "!", "(", ")", "[", "]"]
        # preds, title_preds = get_topic_result(input_title, input_domain)
        # preds, title_preds = get_llm_result(input_title, input_domain)
        preds, title_preds = get_llm_result(input_sys_prompt_str, input_history_str, prompt_str)
        output_data = {"system_prompt": input_sys_prompt_str, "history": input_history_str, "USER": prompt_str,  "ASSISTANT": preds}
        return output_data

    return model_api


#model_path = "/workspaceblobstore/caxu/trained_models/13Bv2_497kcontinueroleplay_dsys_2048_e4_2e_5/checkpoint-75"
#llm = LLM(model=model_path, tensor_parallel_size=1)
# config = Config()
# model  = NERModel(config)