Hugging Face's logo

Spaces:
Tristan
/
static-rlhf-interface
Runtime error

App Files Community
static-rlhf-interface
File size: 4,431 Bytes 
dbe89c5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
import argparse
from os import path

import boto3
from boto.mturk.question import ExternalQuestion
from config import MTURK_KEY, MTURK_SECRET

parser = argparse.ArgumentParser()
parser.add_argument("--mturk_region", default="us-east-1", help="The region for mturk (default: us-east-1)")
parser.add_argument(
    "--space_name",
    default="Tristan/static-rlhf-interface",
    help="Name of the accompanying Hugging Face space (default: huggingface/rlhf-interface)",
)
parser.add_argument("--num_hits", type=int, default=5, help="The number of HITs.")
parser.add_argument(
    "--num_assignments", type=int, default=1, help="The number of times that the HIT can be accepted and completed."
)
parser.add_argument(
    "--live_mode",
    action="store_true",
    help="""
    Whether to run in live mode with real turkers. This will charge your account money.
    If you don't use this flag, the HITs will be deployed on the sandbox version of mturk,
    which will not charge your account money.
    """,
)
parser.add_argument(
    "--refresh_qualification_test",
    action="store_true",
    help="""
    Whether to refresh the qualification test. If you've made edits to the test
    xml files, it is necessary to do this.
    """,
)
parser.add_argument(
    "--custom_qualification_test",
    action="store_true",
    help="""
    Whether to require the custom qualification test.
    """,
)
parser.add_argument(
    "--master_turkers",
    action="store_true",
    help="""
    Whether to only use turkers with the master qualification.
    """,
)
parser.add_argument(
    "--us_turkers",
    action="store_true",
    help="""
    Whether to only use US-based turkers.
    """,
)

args = parser.parse_args()

MTURK_URL = f"https://mturk-requester{'' if args.live_mode else '-sandbox'}.{args.mturk_region}.amazonaws.com"

mturk = boto3.client(
    "mturk",
    aws_access_key_id=MTURK_KEY,
    aws_secret_access_key=MTURK_SECRET,
    region_name=args.mturk_region,
    endpoint_url=MTURK_URL,
)

# This is the URL that makes the space embeddable in an mturk iframe
question = ExternalQuestion(f"https://hf.space/embed/{args.space_name}/+?__theme=light", frame_height=600)

qualification_requirements=[]

if args.master_turkers:
    qualification_requirements.append({
        QualificationTypeId: '2F1QJWKUDD8XADTFD2Q0G6UTO95ALH',
        Comparator: 'Exists'
      })

if args.us_turkers:
    qualification_requirements.append({
        QualificationTypeId: '00000000000000000071',
        Comparator: 'In',
        LocaleValues: [
          { Country: "US" },
        ]
      })

if args.custom_qualification_test:
    qualification_type_id = (
        open("qualification_type_id.txt", "r").read() if path.exists("qualification_type_id.txt") else None
    )
    if args.refresh_qualification_test or qualification_type_id is None:
        if qualification_type_id is not None:
            mturk.delete_qualification_type(QualificationTypeId=qualification_type_id)
        response = mturk.create_qualification_type(
            Name="rlhf--qualification",
            Keywords="RLHF qualification",
            Description="Qualification test for RLHF task.",
            QualificationTypeStatus="Active",
            Test=open("qualification_questions.xml", mode="r").read(),
            AnswerKey=open("qualification_answers.xml", mode="r").read(),
            TestDurationInSeconds=3600,
            AutoGranted=False,
        )
        qualification_type_id = response["QualificationType"]["QualificationTypeId"]
        open("qualification_type_id.txt", "w+").write(qualification_type_id)
    qualification_requirements.append({
            "QualificationTypeId": qualification_type_id,
            "Comparator": "Exists",
            "RequiredToPreview": False,
            "ActionsGuarded": "Accept",
        })

for i in range(args.num_hits):
    new_hit = mturk.create_hit(
        Title="RLHF HIT",
        Description="Interact with an AI",
        Keywords="chatbot",
        Reward="0.25",
        MaxAssignments=args.num_assignments,
        LifetimeInSeconds=172800,
        AssignmentDurationInSeconds=600,
        AutoApprovalDelayInSeconds=14400,
        Question=question.get_as_xml(),
        QualificationRequirements=qualification_requirements,
    )

print(
    f"HIT Group Link: https://worker{'' if args.live_mode else 'sandbox'}.mturk.com/mturk/preview?groupId="
    + new_hit["HIT"]["HITGroupId"]
)