Spaces:
Running
Running
| import json | |
| import requests | |
| PREREQUISITE_PROMPT = """\ | |
| あなたは採点者です。 | |
| 問題, 採点基準, 回答 が与えられます。 | |
| 回答を1,2,3,4,5の5段階で採点し、数字のみを出力してください。 | |
| # 採点基準 | |
| 基本的な採点基準 | |
| - 1点: 誤っている、 指示に従えていない | |
| - 2点: 誤っているが、方向性は合っている | |
| - 3点: 部分的に誤っている、 部分的に合っている | |
| - 4点: 合っている | |
| - 5点: 役に立つ | |
| 基本的な減点項目 | |
| - 不自然な日本語: -1点 | |
| - 部分的に事実と異なる内容を述べている: -1点 | |
| """ | |
| def evaluation_prompt( | |
| input: str, output: str, eval_aspect: str | None, target: str | None | |
| ) -> str: | |
| return f"""\ | |
| 回答を1,2,3,4,5の5段階で採点し、数字のみを出力してください。 | |
| # 問題: {input} | |
| {f"# 正解例: {target}" if target is not None else ""} | |
| {f"# 採点基準: {eval_aspect}" if eval_aspect is not None else ""} | |
| # 回答: {output} | |
| """ | |
| # GradioからのGemini SDKを用いた通信がいつまでも終わらないため、REST APIを利用する | |
| def evaluate(results: list[dict], api_key: str, batch_size: int = 10) -> list[dict]: | |
| url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash:generateContent?key={api_key}" | |
| headers = {"Content-Type": "application/json"} | |
| evaluations = [] | |
| for i in range(0, len(results), batch_size): | |
| batch_results = results[i : i + batch_size] | |
| prompts = [ | |
| evaluation_prompt( | |
| result["input"], | |
| result["output"], | |
| result.get("eval_aspect"), | |
| result.get("target"), | |
| ) | |
| for result in batch_results | |
| ] | |
| data = { | |
| "contents": [{"parts": [{"text": "\n".join(prompts)}]}], | |
| "generationConfig": { | |
| "response_mime_type": "application/json", | |
| "response_schema": {"type": "ARRAY", "items": {"type": "NUMBER"}}, | |
| }, | |
| } | |
| response = requests.post(url, headers=headers, data=json.dumps(data)) | |
| if response.status_code == 200: | |
| response_data = response.json() | |
| # Parse the response_data to extract the scores | |
| scores = json.loads( | |
| response_data["candidates"][0]["content"]["parts"][0]["text"] | |
| ) | |
| else: | |
| raise Exception( | |
| f"API request failed with status code {response.status_code}: {response.text}" | |
| ) | |
| for result, score in zip(batch_results, scores): | |
| evaluation = { | |
| "input": result["input"], | |
| "output": result["output"], | |
| "score": score, | |
| } | |
| if "eval_aspect" in result: | |
| evaluation["eval_aspect"] = result["eval_aspect"] | |
| if "target" in result: | |
| evaluation["target"] = result["target"] | |
| evaluations.append(evaluation) | |
| return evaluations | |
| def report(tasks: list[dict]) -> str: | |
| return ( | |
| """\ | |
| <!DOCTYPE html> | |
| <html lang="ja"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>レポート</title> | |
| <style> | |
| body { | |
| background-color: #f8f9fa; | |
| } | |
| .container { | |
| width: 80%; /* 可変幅 */ | |
| margin: 20px auto; | |
| background-color: #ffffff; | |
| border-radius: 8px; | |
| } | |
| .divider { | |
| position: relative; | |
| padding: 16px 0; | |
| align-items: center; | |
| justify-content: center; | |
| } | |
| .divider .line { | |
| height: 1px; | |
| background-color: #ddd; | |
| } | |
| .divider .taskName { | |
| position: absolute; | |
| margin: -8px; | |
| left: 50%; | |
| transform: translateX(-50%); | |
| padding: 0 10px; | |
| font-size: 14px; | |
| font-weight: 900; | |
| text-align: center; | |
| border: 1px solid #ddd; | |
| border-radius: 9999px; | |
| background-color: #ffffff; | |
| white-space: nowrap; | |
| } | |
| .message { | |
| padding: 8px; | |
| } | |
| .content { | |
| font-size: 14px; | |
| font-weight: 400; | |
| } | |
| .from { | |
| font-size: 14px; | |
| font-weight: 900; | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="container" id="container"></div> | |
| <script> | |
| const messages = """ | |
| + json.dumps(tasks) | |
| + """; | |
| // taskName: str | |
| const createDivider = (taskName) => { | |
| const divider = document.createElement('div'); | |
| divider.classList.add('divider'); | |
| const line = document.createElement('div'); | |
| line.classList.add('line'); | |
| const taskNameLabel = document.createElement('div'); | |
| taskNameLabel.classList.add('taskName'); | |
| taskNameLabel.textContent = taskName; | |
| divider.appendChild(line); | |
| divider.appendChild(taskNameLabel); | |
| return divider; | |
| }; | |
| // task: HTMLDivElement, from: 'input' | 'output' | str, text: string | |
| // return: HTMLDivElement | |
| const createMessage = (text, name) => { | |
| const message = document.createElement('div'); | |
| message.classList.add('message'); | |
| const from = document.createElement('div'); | |
| from.classList.add('from'); | |
| from.textContent = name; | |
| const content = document.createElement('div'); | |
| content.classList.add('content'); | |
| content.innerHTML = text.replace(/\\n/g, '<br>'); | |
| message.appendChild(from); | |
| message.appendChild(content); | |
| return message; | |
| }; | |
| const container = document.getElementById('container'); | |
| messages.forEach((message, i) => { | |
| const task = document.createElement('div'); | |
| task.classList.add('task'); | |
| task.appendChild(createDivider(message.task_id ? `Task ID: ${message.task_id}` : `Task Index ${i}`)); | |
| task.appendChild(createMessage(message.input, 'input')); | |
| task.appendChild(createMessage(message.output, 'output' + (message.score ? ` (score: ${message.score})` : ''))); | |
| container.appendChild(task); | |
| }); | |
| </script> | |
| </body> | |
| </html> | |
| """ | |
| ) | |