Spaces:
Running
Running
import os | |
from hackaprompt.completers import get_completer | |
from hackaprompt.evaluator import get_evaluator | |
from hackaprompt.utils import contains_blacklisted_word, random_alphanumeric, remove_whitespace | |
def test_remove_whitespace(): | |
test_string = "\n\n Hello, world! \n\n" | |
output = remove_whitespace(test_string) | |
expected_output = "Hello, world!" | |
assert expected_output == output | |
def test_random_alphanumeric(): | |
output = random_alphanumeric(k=6) | |
assert isinstance(output, str) | |
assert len(output) == 6 | |
def test_remove_whitespace_alphanumeric(): | |
expected_output = random_alphanumeric(k=6) | |
output = remove_whitespace(expected_output) | |
assert expected_output == output | |
def test_contains_blacklisted_word(): | |
assert contains_blacklisted_word("this string contains the word PWNED") is True | |
assert contains_blacklisted_word("this string contains the word PawNeD") is True | |
assert contains_blacklisted_word("this string will pass the blacklist test.") is False | |
assert contains_blacklisted_word("this is also valid p-wn-ed.") is False | |
def test_all_levels_fail(): | |
"""This test runs every level, for every completer. The result is always incorrect, however it should never throw an error.""" | |
openai_api_key = os.getenv("OPENAI_API_KEY") | |
for level in range(8): | |
for model in ["gpt-3.5-turbo", "FlanT5-XXL"]: | |
completer = get_completer(model, openai_api_key) | |
evaluator = get_evaluator(level=level, completer=completer) | |
prompt = """This is a dummy prompt that will fail the evaluation.""" | |
response = evaluator.evaluate(prompt) | |
assert response.correct is False | |
# we shouldn't have hit an error | |
assert response.error is False | |