WIP: Intended Purpose check

app.py

@@ -6,6 +6,7 @@ from compliance_checks import (
     IntendedPurposeCheck
 )
 
+
 def run_compliance_check(repo_name):
     model_card = ModelCard.load(repo_id_or_path=repo_name).content
 

compliance_checks.py

@@ -25,27 +25,37 @@ class ModelProviderIdentityCheck(ComplianceCheck):
             return False, None
 
 
-class IntendedPurposeCheck(ComplianceCheck):
-    def run_check(self, card: BeautifulSoup):
-        try:
-            direct_use = card.find("h3", string="Direct Use")
+def walk_to_next_heading(card, heading, heading_text):
+    try:
+        heading_node = card.find(heading, string=heading_text)
 
-            direct_use_content = ""
+        content = ""
 
-            sibling_gen = direct_use.nextSiblingGenerator()
+        sibling_gen = heading_node.nextSiblingGenerator()
+        sibling = next(sibling_gen)
+
+        while not (sibling.name is not None and sibling.name.startswith("h")) or sibling.name is None:
+            if not isinstance(sibling, Comment):
+                content = content + sibling.text.strip()
             sibling = next(sibling_gen)
 
-            while sibling.name != "h3":
-                if not isinstance(sibling, Comment):
-                    direct_use_content = direct_use_content + sibling.text
-                sibling = next(sibling_gen)
+        if content.strip() == "[More Information Needed]":
+            return False, None
 
-            if direct_use_content.strip() == "[More Information Needed]":
-                return False, None
+        return True, content
+    except AttributeError:
+        return False, None
 
-            return True, None
-        except AttributeError:
-            return False, None
+
+class IntendedPurposeCheck(ComplianceCheck):
+    def run_check(self, card: BeautifulSoup):
+        direct_use_check, direct_use_content = walk_to_next_heading(card, "h3", "Direct Use")
+        downstream_use_check, downstream_use_content = walk_to_next_heading(card, "h3", "Downstream Use [optional]")
+        out_of_scope_use_check, out_of_scope_use_content = walk_to_next_heading(card, "h3", "Out-of-Scope Use")
+        return (
+            direct_use_check and out_of_scope_use_check,
+            [direct_use_content, downstream_use_content, out_of_scope_use_content]
+        )
 
 
 class ComplianceSuite:

tests/test_compliance_checks.py

@@ -52,8 +52,6 @@ Some random info...
 
 ### Direct Use
 
-<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
-
 Here is some info about direct uses...
 
 ### Downstream Use [optional]
@@ -66,7 +64,7 @@ Here is some info about direct uses...
 
 <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
 
-[More Information Needed]
+Here is some info about out-of-scope uses...
 
 ## Bias, Risks, and Limitations
 
@@ -114,8 +112,8 @@ Some random info...
     @pytest.mark.parametrize("check, card,check_passed,values", [
         (ModelProviderIdentityCheck(), "provider_identity_model_card", True, "Nima Boscarino"),
         (ModelProviderIdentityCheck(), "bad_provider_identity_model_card", False, None),
-        (IntendedPurposeCheck(), "intended_purpose_model_card", True, None),
-        (IntendedPurposeCheck(), "bad_intended_purpose_model_card", False, None),
+        (IntendedPurposeCheck(), "intended_purpose_model_card", True, ["Here is some info about direct uses...", None, "Here is some info about out-of-scope uses..."]),
+        (IntendedPurposeCheck(), "bad_intended_purpose_model_card", False, [None, None, None]),
     ])
     def test_run_model_provider_identity_check(self, check, card, check_passed, values, request):
         card = request.getfixturevalue(card)