Spaces:
Running
Running
File size: 1,781 Bytes
e4f691b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
import os
from hackaprompt.completers import get_completer
from hackaprompt.evaluator import get_evaluator
from hackaprompt.utils import contains_blacklisted_word, random_alphanumeric, remove_whitespace
def test_remove_whitespace():
test_string = "\n\n Hello, world! \n\n"
output = remove_whitespace(test_string)
expected_output = "Hello, world!"
assert expected_output == output
def test_random_alphanumeric():
output = random_alphanumeric(k=6)
assert isinstance(output, str)
assert len(output) == 6
def test_remove_whitespace_alphanumeric():
expected_output = random_alphanumeric(k=6)
output = remove_whitespace(expected_output)
assert expected_output == output
def test_contains_blacklisted_word():
assert contains_blacklisted_word("this string contains the word PWNED") is True
assert contains_blacklisted_word("this string contains the word PawNeD") is True
assert contains_blacklisted_word("this string will pass the blacklist test.") is False
assert contains_blacklisted_word("this is also valid p-wn-ed.") is False
def test_all_levels_fail():
"""This test runs every level, for every completer. The result is always incorrect, however it should never throw an error."""
openai_api_key = os.getenv("OPENAI_API_KEY")
for level in range(8):
for model in ["gpt-3.5-turbo", "FlanT5-XXL"]:
completer = get_completer(model, openai_api_key)
evaluator = get_evaluator(level=level, completer=completer)
prompt = """This is a dummy prompt that will fail the evaluation."""
response = evaluator.evaluate(prompt)
assert response.correct is False
# we shouldn't have hit an error
assert response.error is False
|