import os import os from pathlib import Path import csv import json import time import pandas as pd import requests from PIL import Image import time from io import BytesIO import json # Set up the OpenAI API client api_key = "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" # Set up the chatGPT model and prompt model_engine = "text-davinci-003" import gradio as gr import time import argparse from vllm import LLM, SamplingParams def call_async_server(prompt): json_object = {"prompt": prompt, # "max_tokens": 2048000, "max_tokens": 2048000, "n": 1 } generate_response = requests.post("http://phlrr3058.guest.corp.microsoft.com:7860/generate", json=json_object) end = time.time() res_json = json.loads(generate_response.content) ASSISTANT = res_json['text'][-1].split("ASSISTANT:")[-1].strip() print(ASSISTANT) return ASSISTANT def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--model", type=str) # model path parser.add_argument("--n_gpu", type=int, default=1) # n_gpu return parser.parse_args() def echo(message, history, system_prompt, temperature, max_tokens): response = f"System prompt: {system_prompt}\n Message: {message}. \n Temperature: {temperature}. \n Max Tokens: {max_tokens}." for i in range(min(len(response), int(max_tokens))): time.sleep(0.05) yield response[: i+1] def align_data(data): """Given dict with lists, creates aligned strings Adapted from Assignment 3 of CS224N Args: data: (dict) data["x"] = ["I", "love", "you"] (dict) data["y"] = ["O", "O", "O"] Returns: data_aligned: (dict) data_align["x"] = "I love you" data_align["y"] = "O O O " """ spacings = [max([len(seq[i]) for seq in data.values()]) for i in range(len(data[list(data.keys())[0]]))] data_aligned = dict() # for each entry, create aligned string for key, seq in data.items(): str_aligned = "" for token, spacing in zip(seq, spacings): str_aligned += token + " " * (spacing - len(token) + 1) data_aligned[key] = str_aligned return data_aligned # def get_llm_result(input_data, input_domain): def get_llm_result(input_sys_prompt_str, input_history_str, prompt_str): # data is file path of topic result prompt = "" def predict(message, history, system_prompt, temperature, max_tokens): # instruction = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. " instruction = system_prompt for human, assistant in history: instruction += 'USER: '+ human + ' ASSISTANT: '+ assistant + '' instruction += 'USER: '+ message + ' ASSISTANT:' # problem = [instruction] # stop_tokens = ["Question:", "Question", "USER:", "USER", "ASSISTANT:", "ASSISTANT", "Instruction:", "Instruction", "Response:", "Response"] # sampling_params = SamplingParams(temperature=temperature, top_p=1, max_tokens=max_tokens, stop=stop_tokens) # completions = llm.generate(problem, sampling_params) generate_rext = call_async_server(instruction) return generate_rext # for output in completions: # prompt = output.prompt # generated_text = output.outputs[0].text # return generated_text # for idx in range(len(generated_text)): # yield generated_text[:idx+1] try: # completion = openai.Completion.create( # engine=model_engine, # prompt=prompt, # max_tokens=3000, # n=1, # stop=None, # temperature=0.5, # ) # # response = completion.choices[0].text # shorten_response = response.replace("\n", "").strip() # len_response = len(shorten_response.split(" ")) # if len_response >= 3500: # shorten_response = "".join(shorten_response.split(" ")[:3500]) # print("X"*10) # print(f"shorten_response is {shorten_response}") # list_shorten = shorten_response.split(" ") # print(list_shorten) # print(f"length is {len(list_shorten)}") # title_prompt = f"{shorten_response}，给这个文章写一个头条号风格的标题。增加标题的吸引力，可读性。" # title_completion = openai.Completion.create( # engine=model_engine, # prompt=title_prompt, # max_tokens=200, # n=1, # stop=None, # temperature=0.5, # ) # title_response = title_completion.choices[0].text # model_path = "/workspaceblobstore/caxu/trained_models/13Bv2_497kcontinueroleplay_dsys_2048_e4_2e_5/checkpoint-75" # llm = LLM(model=model_path, tensor_parallel_size=1) history = input_history_str prompt = prompt_str system_prompt = input_sys_prompt_str response = predict(prompt, history, system_prompt, 0.5, 3000) print(response) # if not os.path.isdir(topic_file_path): # print("File folder not exist") # topic_result_file = "" # topic_file_name_pattern = "step10_json_filestep9_merge_rewrite_" # for filename in os.listdir(topic_file_path): # if filename.startswith(topic_file_name_pattern): # topic_result_file = os.path.join(topic_file_path, filename) # # data_aligned = dict() # output_dir_name = "." # output_dir = os.path.join(output_dir_name, "result_topic_file") # Path(output_dir).mkdir(parents=True, exist_ok=True) # write_file_name = "save_server_" + topic_file_path.split("\\")[-1] # write_output_file_path = os.path.join(output_dir, write_file_name) # # with open(topic_result_file, encoding="utf8") as f: # json_data = json.load(f) # return json_data return response, response except Exception as ex: print("File not exist") raise ex def get_model_api(): """Returns lambda function for api""" # def model_api(input_title, input_domain): def model_api(input_sys_prompt_str, input_history_str, prompt_str): """ Args: input_data: submitted to the API, raw string Returns: output_data: after some transformation, to be returned to the API """ # print("X"*10) # print(f"input_title is {input_title}") # print(f"input_data2 is {input_domain}") punc = [",", "?", ".", ":", ";", "!", "(", ")", "[", "]"] # preds, title_preds = get_topic_result(input_title, input_domain) # preds, title_preds = get_llm_result(input_title, input_domain) preds, title_preds = get_llm_result(input_sys_prompt_str, input_history_str, prompt_str) output_data = {"system_prompt": input_sys_prompt_str, "history": input_history_str, "USER": prompt_str, "ASSISTANT": preds} return output_data return model_api #model_path = "/workspaceblobstore/caxu/trained_models/13Bv2_497kcontinueroleplay_dsys_2048_e4_2e_5/checkpoint-75" #llm = LLM(model=model_path, tensor_parallel_size=1) # config = Config() # model = NERModel(config)