Spaces:

MLCommons
/

croissant-editor

Running

App Files Files Community

marcenacp commited on Dec 6, 2023

Commit

bc133ae

1 Parent(s): 8c11dd4

Deploy (see actual commits on https://github.com/mlcommons/croissant).

Browse files

Files changed (20) hide show

components/tabs/frontend/build/asset-manifest.json +3 -3
components/tabs/frontend/build/index.html +1 -1
components/tabs/frontend/build/static/js/main.a44b10fc.js +0 -0
components/tabs/frontend/build/static/js/main.a44b10fc.js.LICENSE.txt +73 -0
components/tabs/frontend/build/static/js/main.a44b10fc.js.map +0 -0
components/tabs/frontend/src/Tabs.tsx +1 -1
core/constants.py +5 -0
core/files.py +1 -1
core/names.py +5 -0
core/names_test.py +1 -0
core/state.py +4 -0
events/metadata.py +14 -1
views/files.py +19 -9
views/load.py +1 -1
views/metadata.py +64 -1
views/overview.py +17 -8
views/previous_files.py +1 -1
views/record_sets.py +31 -14
views/source.py +46 -11
views/splash.py +43 -6

components/tabs/frontend/build/asset-manifest.json CHANGED Viewed

@@ -1,10 +1,10 @@
 {
   "files": {
-    "main.js": "./static/js/main.e6b754d8.js",
     "index.html": "./index.html",
-    "main.e6b754d8.js.map": "./static/js/main.e6b754d8.js.map"
   },
   "entrypoints": [
-    "static/js/main.e6b754d8.js"
   ]
 }

 {
   "files": {
+    "main.js": "./static/js/main.a44b10fc.js",
     "index.html": "./index.html",
+    "main.a44b10fc.js.map": "./static/js/main.a44b10fc.js.map"
   },
   "entrypoints": [
+    "static/js/main.a44b10fc.js"
   ]
 }

components/tabs/frontend/build/index.html CHANGED Viewed

@@ -1 +1 @@

- <!doctype html><html lang="en"><head><title>Streamlit Tabs Component</title><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Streamlit Tree Component"/><script defer="defer" src="./static/js/main.~~e6b754d8~~.js"></script></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

+ <!doctype html><html lang="en"><head><title>Streamlit Tabs Component</title><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Streamlit Tree Component"/><script defer="defer" src="./static/js/main.a44b10fc.js"></script></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

components/tabs/frontend/build/static/js/main.a44b10fc.js ADDED Viewed

The diff for this file is too large to render. See raw diff

components/tabs/frontend/build/static/js/main.a44b10fc.js.LICENSE.txt ADDED Viewed

	@@ -0,0 +1,73 @@

+/*
+object-assign
+(c) Sindre Sorhus
+@license MIT
+*/
+/**
+ * @license React
+ * react-dom.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/**
+ * @license React
+ * react-is.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/**
+ * @license React
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/**
+ * @license React
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/**
+ * @license React
+ * scheduler.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/** @license React v16.13.1
+ * react-is.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/** @license React v16.14.0
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */

components/tabs/frontend/build/static/js/main.a44b10fc.js.map ADDED Viewed

The diff for this file is too large to render. See raw diff

components/tabs/frontend/src/Tabs.tsx CHANGED Viewed

@@ -84,7 +84,7 @@ function BasicTabs({
               whiteSpace: "nowrap",
             }}
           >
-            Download 🥐 file
           </Button>
         </span>
       </Tooltip>

               whiteSpace: "nowrap",
             }}
           >
+            Export
           </Button>
         </span>
       </Tooltip>

core/constants.py CHANGED Viewed

@@ -35,3 +35,8 @@ METADATA = "Metadata"
 RESOURCES = "Resources"
 RECORD_SETS = "Record Sets"
 TABS = [OVERVIEW, METADATA, RESOURCES, RECORD_SETS]

 RESOURCES = "Resources"
 RECORD_SETS = "Record Sets"
 TABS = [OVERVIEW, METADATA, RESOURCES, RECORD_SETS]
+NAMES_INFO = (
+    "Names are used as identifiers. They are unique and cannot contain special"
+    " characters. The interface will replace any special characters."
+)

core/files.py CHANGED Viewed

@@ -204,7 +204,7 @@ def file_from_form(
     if type == FILE_OBJECT:
         return FileObject(name=find_unique_name(names, "file_object"), folder=folder)
     elif type == FILE_SET:
-        return FileSet(name=find_unique_name(names, "file_set"), folder=folder)
     else:
         raise ValueError("type has to be one of FILE_OBJECT, FILE_SET")

     if type == FILE_OBJECT:
         return FileObject(name=find_unique_name(names, "file_object"), folder=folder)
     elif type == FILE_SET:
+        return FileSet(name=find_unique_name(names, "file_set"))
     else:
         raise ValueError("type has to be one of FILE_OBJECT, FILE_SET")

core/names.py CHANGED Viewed

@@ -1,8 +1,13 @@
 """Module to handle naming of RecordSets and distribution."""
 def find_unique_name(names: set[str], name: str):
     """Find a unique UID."""
     while name in names:
         name = f"{name}_0"
     return name

 """Module to handle naming of RecordSets and distribution."""
+import re
+NAME_PATTERN_REGEX = "[^a-zA-Z0-9\\-_\\.]"
 def find_unique_name(names: set[str], name: str):
     """Find a unique UID."""
+    name = re.sub(NAME_PATTERN_REGEX, "_", name)
     while name in names:
         name = f"{name}_0"
     return name

core/names_test.py CHANGED Viewed

@@ -5,6 +5,7 @@ from .names import find_unique_name
 def test_find_unique_name():
     names = set(["first", "second", "first_0"])
     assert find_unique_name(names, "first") == "first_0_0"
     assert find_unique_name(names, "second") == "second_0"
     assert find_unique_name(names, "third") == "third"

 def test_find_unique_name():
     names = set(["first", "second", "first_0"])
+    assert find_unique_name(names, "are there spaces") == "are_there_spaces"
     assert find_unique_name(names, "first") == "first_0_0"
     assert find_unique_name(names, "second") == "second_0"
     assert find_unique_name(names, "third") == "third"

core/state.py CHANGED Viewed

@@ -183,11 +183,15 @@ class Metadata:
     name: str = ""
     description: str | None = None
     citation: str | None = None
     license: str | None = ""
     url: str = ""
     distribution: list[FileObject | FileSet] = dataclasses.field(default_factory=list)
     record_sets: list[RecordSet] = dataclasses.field(default_factory=list)
     rdf: mlc.Rdf = dataclasses.field(default_factory=mlc.Rdf)
     def __bool__(self):
         return self.name != "" and self.url != ""

     name: str = ""
     description: str | None = None
     citation: str | None = None
+    data_biases: str | None = None
+    data_collection: str | None = None
     license: str | None = ""
+    personal_sensitive_information: str | None = None
     url: str = ""
     distribution: list[FileObject | FileSet] = dataclasses.field(default_factory=list)
     record_sets: list[RecordSet] = dataclasses.field(default_factory=list)
     rdf: mlc.Rdf = dataclasses.field(default_factory=mlc.Rdf)
+    version: str | None = None
     def __bool__(self):
         return self.name != "" and self.url != ""

events/metadata.py CHANGED Viewed

@@ -2,6 +2,7 @@ import enum
 import streamlit as st
 from core.state import Metadata
 # List from:
@@ -93,11 +94,15 @@ class MetadataEvent(enum.Enum):
     URL = "URL"
     LICENSE = "LICENSE"
     CITATION = "CITATION"
 def handle_metadata_change(event: MetadataEvent, metadata: Metadata, key: str):
     if event == MetadataEvent.NAME:
-        metadata.name = st.session_state[key]
     elif event == MetadataEvent.DESCRIPTION:
         metadata.description = st.session_state[key]
     elif event == MetadataEvent.LICENSE:
@@ -106,3 +111,11 @@ def handle_metadata_change(event: MetadataEvent, metadata: Metadata, key: str):
         metadata.citation = st.session_state[key]
     elif event == MetadataEvent.URL:
         metadata.url = st.session_state[key]

 import streamlit as st
+from core.names import find_unique_name
 from core.state import Metadata
 # List from:
     URL = "URL"
     LICENSE = "LICENSE"
     CITATION = "CITATION"
+    VERSION = "VERSION"
+    DATA_BIASES = "DATA_BIASES"
+    DATA_COLLECTION = "DATA_COLLECTION"
+    PERSONAL_SENSITIVE_INFORMATION = "PERSONAL_SENSITIVE_INFORMATION"
 def handle_metadata_change(event: MetadataEvent, metadata: Metadata, key: str):
     if event == MetadataEvent.NAME:
+        metadata.name = find_unique_name(set(), st.session_state[key])
     elif event == MetadataEvent.DESCRIPTION:
         metadata.description = st.session_state[key]
     elif event == MetadataEvent.LICENSE:
         metadata.citation = st.session_state[key]
     elif event == MetadataEvent.URL:
         metadata.url = st.session_state[key]
+    elif event == MetadataEvent.VERSION:
+        metadata.version = st.session_state[key]
+    elif event == MetadataEvent.DATA_BIASES:
+        metadata.data_biases = st.session_state[key]
+    elif event == MetadataEvent.DATA_COLLECTION:
+        metadata.data_collection = st.session_state[key]
+    elif event == MetadataEvent.PERSONAL_SENSITIVE_INFORMATION:
+        metadata.personal_sensitive_information = st.session_state[key]

views/files.py CHANGED Viewed

@@ -3,6 +3,7 @@ import streamlit as st
 from components.safe_button import button_with_confirmation
 from components.tree import render_tree
 from core.constants import DF_HEIGHT
 from core.constants import OAUTH_CLIENT_ID
 from core.files import code_to_index
 from core.files import file_from_form
@@ -39,16 +40,15 @@ resources on the web or manually create new resources."""
 def render_files():
     """Renders the views of the files: warnings and panels to display information."""
     _render_warnings()
-    col1, col2, col3 = st.columns([1, 1, 1], gap="small")
     with col1:
-        st.markdown("##### Upload more resources")
         _render_upload_panel()
-    with col2:
         st.markdown("##### Uploaded resources")
         files = st.session_state[Metadata].distribution
         resource = _render_resources_panel(files)
         st.session_state[SelectedResource] = resource
-    with col3:
         _render_right_panel()
@@ -111,9 +111,7 @@ def _render_resources_panel(files: list[Resource]) -> Resource | None:
 def _render_upload_panel():
     """Renders the form to upload from local or upload from URL."""
     with st.form(key="upload_form", clear_on_submit=True):
-        tab1, tab2, tab3 = st.tabs([
-            "Import from a local file", "Import from a URL", "Add manually"
-        ])
         with tab1:
             st.file_uploader("Select a file", key=_LOCAL_FILE_KEY)
@@ -202,6 +200,11 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         default=file.contained_in,
         options=parent_options,
         key=key,
         on_change=handle_resource_change,
         args=(ResourceEvent.CONTAINED_IN, file, key),
     )
@@ -210,6 +213,7 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         needed_field("Name"),
         value=file.name,
         key=key,
         on_change=handle_resource_change,
         args=(ResourceEvent.NAME, file, key),
     )
@@ -217,7 +221,7 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
     st.text_area(
         "Description",
         value=file.description,
-        placeholder="Provide a clear description of the file.",
         key=key,
         on_change=handle_resource_change,
         args=(ResourceEvent.DESCRIPTION, file, key),
@@ -225,9 +229,10 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
     if is_file_object:
         key = f"{prefix}_content_url"
         st.text_input(
-            needed_field("Content URL"),
             value=file.content_url,
             key=key,
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_URL, file, key),
         )
@@ -244,6 +249,7 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
             "Content size",
             value=file.content_size,
             key=key,
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_SIZE, file, key),
         )
@@ -262,6 +268,10 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         index=code_to_index(file.encoding_format),
         options=FILE_TYPES.keys(),
         key=key,
         on_change=handle_resource_change,
         args=(ResourceEvent.ENCODING_FORMAT, file, key),
     )

 from components.safe_button import button_with_confirmation
 from components.tree import render_tree
 from core.constants import DF_HEIGHT
+from core.constants import NAMES_INFO
 from core.constants import OAUTH_CLIENT_ID
 from core.files import code_to_index
 from core.files import file_from_form
 def render_files():
     """Renders the views of the files: warnings and panels to display information."""
     _render_warnings()
+    col1, col2 = st.columns([1, 1], gap="small")
     with col1:
+        st.markdown("##### Add a resource")
         _render_upload_panel()
         st.markdown("##### Uploaded resources")
         files = st.session_state[Metadata].distribution
         resource = _render_resources_panel(files)
         st.session_state[SelectedResource] = resource
+    with col2:
         _render_right_panel()
 def _render_upload_panel():
     """Renders the form to upload from local or upload from URL."""
     with st.form(key="upload_form", clear_on_submit=True):
+        tab1, tab2, tab3 = st.tabs(["From a local file", "From a URL", "Add manually"])
         with tab1:
             st.file_uploader("Select a file", key=_LOCAL_FILE_KEY)
         default=file.contained_in,
         options=parent_options,
         key=key,
+        help=(
+            "FileObjects and FileSets can be nested. Specifying `Parents` allows to"
+            " nest a FileObject/FileSet within another FileObject/FileSet. An example"
+            " of this is when images (FileSet) are nested within an archive (FileSet)."
+        ),
         on_change=handle_resource_change,
         args=(ResourceEvent.CONTAINED_IN, file, key),
     )
         needed_field("Name"),
         value=file.name,
         key=key,
+        help=f"The name of the resource. {NAMES_INFO}",
         on_change=handle_resource_change,
         args=(ResourceEvent.NAME, file, key),
     )
     st.text_area(
         "Description",
         value=file.description,
+        placeholder="Provide a description of the file.",
         key=key,
         on_change=handle_resource_change,
         args=(ResourceEvent.DESCRIPTION, file, key),
     if is_file_object:
         key = f"{prefix}_content_url"
         st.text_input(
+            needed_field("Content URL or local path"),
             value=file.content_url,
             key=key,
+            help="The URL or local file path pointing to the original FileObject.",
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_URL, file, key),
         )
             "Content size",
             value=file.content_size,
             key=key,
+            help="The size of the original FileObject in bytes.",
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_SIZE, file, key),
         )
         index=code_to_index(file.encoding_format),
         options=FILE_TYPES.keys(),
         key=key,
+        help=(
+            "MIME type corresponding to"
+            " ([sc:encodingFormat](https://schema.org/encodingFormat))."
+        ),
         on_change=handle_resource_change,
         args=(ResourceEvent.ENCODING_FORMAT, file, key),
     )

views/load.py CHANGED Viewed

@@ -30,5 +30,5 @@ def _on_file_upload(key):
 def render_load():
     key = "json-ld-file-upload"
     st.file_uploader(
-        "Select a JSON-LD", type="json", key=key, on_change=_on_file_upload, args=(key,)
     )

 def render_load():
     key = "json-ld-file-upload"
     st.file_uploader(
+        "Drop a JSON-LD", type="json", key=key, on_change=_on_file_upload, args=(key,)
     )

views/metadata.py CHANGED Viewed

@@ -10,7 +10,57 @@ from events.metadata import MetadataEvent
 def render_metadata():
     """Renders the `Metadata` view."""
-    metadata = st.session_state[Metadata]
     index = find_license_index(metadata.license)
     key = "metadata-url"
     st.text_input(
@@ -21,6 +71,19 @@ def render_metadata():
         on_change=handle_metadata_change,
         args=(MetadataEvent.URL, metadata, key),
     )
     key = "metadata-license"
     st.selectbox(
         label="License",

 def render_metadata():
     """Renders the `Metadata` view."""
+    metadata: Metadata = st.session_state[Metadata]
+    col1, col2 = st.columns([1, 1])
+    with col1.expander("**Generic metadata**", expanded=True):
+        _render_generic_metadata(metadata)
+    with col2.expander("**Responsible AI (RAI) metadata**", expanded=True):
+        _render_rai_metadata(metadata)
+def _render_rai_metadata(metadata: Metadata):
+    """Renders RAI (Responsible AI) metadata."""
+    key = "metadata-data-collection"
+    st.text_area(
+        label=(
+            "**Data collection**. Key stages of the data collection process encourage"
+            " its creators to reflect on the process and improves understanding for"
+            " users."
+        ),
+        key=key,
+        value=metadata.data_collection,
+        on_change=handle_metadata_change,
+        args=(MetadataEvent.DATA_COLLECTION, metadata, key),
+    )
+    key = "metadata-data-biases"
+    st.text_area(
+        label=(
+            "**Data biases**. Involves understanding the potential risks associated"
+            " with data usage and to prevent unintended and potentially harmful"
+            " consequences that may arise from using models trained on or evaluated"
+            " with the respective data."
+        ),
+        key=key,
+        value=metadata.data_biases,
+        on_change=handle_metadata_change,
+        args=(MetadataEvent.DATA_BIASES, metadata, key),
+    )
+    key = "metadata-personal-sensitive-information"
+    st.text_area(
+        label=(
+            "**Personal sensitive information**. Personal and sensitive information, if"
+            " contained within the dataset, can play an important role in the"
+            " mitigation of any risks and the responsible use of the datasets."
+        ),
+        key=key,
+        value=metadata.personal_sensitive_information,
+        on_change=handle_metadata_change,
+        args=(MetadataEvent.PERSONAL_SENSITIVE_INFORMATION, metadata, key),
+    )
+def _render_generic_metadata(metadata: Metadata):
+    """Renders all non-RAI generic metadata."""
     index = find_license_index(metadata.license)
     key = "metadata-url"
     st.text_input(
         on_change=handle_metadata_change,
         args=(MetadataEvent.URL, metadata, key),
     )
+    key = "metadata-version"
+    st.text_input(
+        label="Version (`MAJOR.MINOR.PATCH`)",
+        key=key,
+        help=(
+            "Refer to https://semver.org/spec/v2.0.0.html for more information on the"
+            " format."
+        ),
+        value=metadata.version,
+        placeholder="1.0.0",
+        on_change=handle_metadata_change,
+        args=(MetadataEvent.VERSION, metadata, key),
+    )
     key = "metadata-license"
     st.selectbox(
         label="License",

views/overview.py CHANGED Viewed

@@ -3,6 +3,7 @@ from typing import Any
 import streamlit as st
 from core.state import Metadata
 import mlcroissant as mlc
 from utils import needed_field
@@ -22,9 +23,9 @@ _INFO_TEXT = """Croissant files are composed of three layers:
                 (typically a file or set of files) and the structure of these records,
                 expressed as a set of fields (e.g., the columns of a table).
-The next three tabs will guide you through filling those layers. The errors if any will
-be displayed on this page. Once you are ready, you can download the dataset by clicking
-the export button in the upper right corner."""
 def _relevant_fields(class_or_instance: type):
@@ -51,6 +52,7 @@ def render_overview():
             label=needed_field("Name"),
             key=key,
             value=metadata.name,
             placeholder="Dataset",
             on_change=handle_metadata_change,
             args=(MetadataEvent.NAME, metadata, key),
@@ -62,7 +64,7 @@ def render_overview():
             label="Description",
             key=key,
             value=metadata.description,
-            placeholder="Provide a clear description of the dataset.",
             on_change=handle_metadata_change,
             args=(MetadataEvent.DESCRIPTION, metadata, key),
         )
@@ -82,10 +84,17 @@ def render_overview():
             * 100
             / (3 * metadata_weight)
         )
-        col_a.metric("Completion", f"{completion}%")
-        col_b.metric("Number of metadata fields", fields)
-        col_c.metric("Number of resources", len(metadata.distribution))
-        col_d.metric("Number of RecordSets", len(metadata.record_sets))
     with col2:
         user_started_editing = metadata.record_sets or metadata.distribution
         if user_started_editing:

 import streamlit as st
+from core.constants import NAMES_INFO
 from core.state import Metadata
 import mlcroissant as mlc
 from utils import needed_field
                 (typically a file or set of files) and the structure of these records,
                 expressed as a set of fields (e.g., the columns of a table).
+The next three tabs will guide you through filling those layers. Any error will be
+displayed on the overview. Once the dataset is finished, you can download the dataset by
+clicking the export button in the upper right corner."""
 def _relevant_fields(class_or_instance: type):
             label=needed_field("Name"),
             key=key,
             value=metadata.name,
+            help=f"The name of the dataset. {NAMES_INFO}",
             placeholder="Dataset",
             on_change=handle_metadata_change,
             args=(MetadataEvent.NAME, metadata, key),
             label="Description",
             key=key,
             value=metadata.description,
+            placeholder="Provide a description of the dataset.",
             on_change=handle_metadata_change,
             args=(MetadataEvent.DESCRIPTION, metadata, key),
         )
             * 100
             / (3 * metadata_weight)
         )
+        col_a.metric(
+            "Completion",
+            f"{completion}%",
+            help=(
+                "Approximation of the total completion based on the number of fields"
+                " that are filled."
+            ),
+        )
+        col_b.metric("Metadata fields", fields)
+        col_c.metric("Resources", len(metadata.distribution))
+        col_d.metric("RecordSets", len(metadata.record_sets))
     with col2:
         user_started_editing = metadata.record_sets or metadata.distribution
         if user_started_editing:

views/previous_files.py CHANGED Viewed

@@ -50,4 +50,4 @@ def render_previous_files():
         except:
             pass
     if has_no_project:
-        st.write("No past project to load. Create one on the left!")

         except:
             pass
     if has_no_project:
+        st.write("No recent project to load. Create one on the left!")

views/record_sets.py CHANGED Viewed

@@ -10,6 +10,7 @@ from rdflib import term
 import streamlit as st
 from components.safe_button import button_with_confirmation
 from core.data_types import MLC_DATA_TYPES
 from core.data_types import mlc_to_str_data_type
 from core.data_types import STR_DATA_TYPES
@@ -240,6 +241,7 @@ def _render_left_panel():
                 needed_field("Name"),
                 placeholder="Name without special character.",
                 key=key,
                 value=record_set.name,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.NAME, record_set, key),
@@ -247,7 +249,7 @@ def _render_left_panel():
             key = f"{prefix}-description"
             col2.text_input(
                 "Description",
-                placeholder="Provide a clear description of the RecordSet.",
                 key=key,
                 value=record_set.description,
                 on_change=handle_record_set_change,
@@ -257,6 +259,13 @@ def _render_left_panel():
             st.checkbox(
                 "The RecordSet is an enumeration",
                 key=key,
                 value=record_set.is_enumeration,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.IS_ENUMERATION, record_set, key),
@@ -265,6 +274,10 @@ def _render_left_panel():
             st.checkbox(
                 "The RecordSet has in-line data",
                 key=key,
                 value=bool(record_set.data),
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.HAS_DATA, record_set, key),
@@ -324,8 +337,14 @@ def _render_left_panel():
             )
             data_editor_key = _data_editor_key(record_set_key, record_set)
             st.markdown(
-                f"{needed_field('Fields')} (add/delete fields by directly editing the"
-                " table)"
             )
             st.data_editor(
                 fields,
@@ -437,6 +456,7 @@ def _render_right_panel():
                     needed_field("Name"),
                     placeholder="Name without special character.",
                     key=key,
                     value=field.name,
                     on_change=handle_field_change,
                     args=(FieldEvent.NAME, field, key),
@@ -444,38 +464,35 @@ def _render_right_panel():
                 key = f"{prefix}-description"
                 col2.text_input(
                     "Description",
-                    placeholder="Provide a clear description of the RecordSet.",
                     key=key,
                     on_change=handle_field_change,
                     value=field.description,
                     args=(FieldEvent.DESCRIPTION, field, key),
                 )
                 if field.data_types:
                     data_type = field.data_types[0]
                     if isinstance(data_type, str):
                         data_type = term.URIRef(data_type)
                     if data_type in MLC_DATA_TYPES:
                         data_type_index = MLC_DATA_TYPES.index(data_type)
-                    else:
-                        data_type_index = None
-                else:
-                    data_type_index = None
                 key = f"{prefix}-datatypes"
                 col3.selectbox(
                     needed_field("Data type"),
                     index=data_type_index,
                     options=STR_DATA_TYPES,
                     key=key,
                     on_change=handle_field_change,
                     args=(FieldEvent.DATA_TYPE, field, key),
                 )
                 possible_sources = _get_possible_sources(metadata)
-                render_source(
-                    record_set_key, record_set, field, field_key, possible_sources
-                )
-                render_references(
-                    record_set_key, record_set, field, field_key, possible_sources
-                )
                 st.divider()

 import streamlit as st
 from components.safe_button import button_with_confirmation
+from core.constants import NAMES_INFO
 from core.data_types import MLC_DATA_TYPES
 from core.data_types import mlc_to_str_data_type
 from core.data_types import STR_DATA_TYPES
                 needed_field("Name"),
                 placeholder="Name without special character.",
                 key=key,
+                help=f"The name of the RecordSet. {NAMES_INFO}",
                 value=record_set.name,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.NAME, record_set, key),
             key = f"{prefix}-description"
             col2.text_input(
                 "Description",
+                placeholder="Provide a description of the RecordSet.",
                 key=key,
                 value=record_set.description,
                 on_change=handle_record_set_change,
             st.checkbox(
                 "The RecordSet is an enumeration",
                 key=key,
+                help=(
+                    "Enumerations indicate that the RecordSet takes its values from a"
+                    " finite set. Similar to `ClassLabel` in"
+                    " [TFDS](https://www.tensorflow.org/datasets/api_docs/python/tfds/features/ClassLabel)"
+                    " or [Hugging"
+                    " Face](https://huggingface.co/docs/datasets/v2.15.0/en/package_reference/main_classes#datasets.ClassLabel)."
+                ),
                 value=record_set.is_enumeration,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.IS_ENUMERATION, record_set, key),
             st.checkbox(
                 "The RecordSet has in-line data",
                 key=key,
+                help=(
+                    "In-line data allows to embed data directly within the JSON-LD"
+                    " without referencing another data source."
+                ),
                 value=bool(record_set.data),
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.HAS_DATA, record_set, key),
             )
             data_editor_key = _data_editor_key(record_set_key, record_set)
             st.markdown(
+                needed_field("Fields"),
+                help=(
+                    "Add/delete fields by directly editing the table. **Warning**: the"
+                    " table contains information about the fields--not the data"
+                    " directly. If you wish to embed data, tick the `The RecordSet is"
+                    " an enumeration` box. To edit fields details, click the"
+                    " button `Edit fields details` below."
+                ),
             )
             st.data_editor(
                 fields,
                     needed_field("Name"),
                     placeholder="Name without special character.",
                     key=key,
+                    help=f"The name of the field. {NAMES_INFO}",
                     value=field.name,
                     on_change=handle_field_change,
                     args=(FieldEvent.NAME, field, key),
                 key = f"{prefix}-description"
                 col2.text_input(
                     "Description",
+                    placeholder="Provide a description of the RecordSet.",
                     key=key,
                     on_change=handle_field_change,
                     value=field.description,
                     args=(FieldEvent.DESCRIPTION, field, key),
                 )
+                data_type_index = None
                 if field.data_types:
                     data_type = field.data_types[0]
                     if isinstance(data_type, str):
                         data_type = term.URIRef(data_type)
                     if data_type in MLC_DATA_TYPES:
                         data_type_index = MLC_DATA_TYPES.index(data_type)
                 key = f"{prefix}-datatypes"
                 col3.selectbox(
                     needed_field("Data type"),
                     index=data_type_index,
                     options=STR_DATA_TYPES,
                     key=key,
+                    help=(
+                        "The type of the data. `Text` corresponds to"
+                        " https://schema.org/Text, etc."
+                    ),
                     on_change=handle_field_change,
                     args=(FieldEvent.DATA_TYPE, field, key),
                 )
                 possible_sources = _get_possible_sources(metadata)
+                render_source(record_set, field, possible_sources)
+                render_references(record_set, field, possible_sources)
                 st.divider()

views/source.py CHANGED Viewed

@@ -12,6 +12,15 @@ from events.fields import TransformType
 import mlcroissant as mlc
 from utils import needed_field
 class SourceType:
     """The type of the source (distribution or field)."""
@@ -105,10 +114,8 @@ def _handle_remove_reference(field):
 def render_source(
-    record_set_key: int,
     record_set: RecordSet,
     field: Field,
-    field_key: int,
     possible_sources: list[str],
 ):
     """Renders the form for the source."""
@@ -123,10 +130,13 @@ def render_source(
         index = None
     key = f"{prefix}-source"
     col1.selectbox(
-        needed_field("Source"),
         index=index,
         options=options,
         key=key,
         on_change=handle_field_change,
         args=(FieldEvent.SOURCE, field, key),
     )
@@ -135,6 +145,7 @@ def render_source(
             needed_field("Extract"),
             index=_get_extract_index(source),
             key=f"{prefix}-extract",
             options=EXTRACT_TYPES,
             on_change=handle_field_change,
             args=(FieldEvent.SOURCE_EXTRACT, field, key),
@@ -145,6 +156,7 @@ def render_source(
                 needed_field("Column name"),
                 value=source.extract.column,
                 key=key,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_COLUMN, field, key),
             )
@@ -154,6 +166,7 @@ def render_source(
                 needed_field("JSON path"),
                 value=source.extract.json_path,
                 key=key,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_JSON_PATH, field, key),
             )
@@ -170,18 +183,23 @@ def render_source(
                 key=key,
                 options=TRANSFORM_TYPES,
                 on_change=handle_field_change,
                 args=(FieldEvent.TRANSFORM, field, key),
                 kwargs={"number": number},
             )
             if selected == TransformType.FORMAT:
                 key = f"{prefix}-{number}-transform-format"
                 col3.text_input(
-                    needed_field("Format"),
                     value=transform.format,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.JSON_PATH:
                 key = f"{prefix}-{number}-jsonpath"
@@ -190,8 +208,9 @@ def render_source(
                     value=transform.json_path,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.REGEX:
                 key = f"{prefix}-{number}-regex"
@@ -200,8 +219,14 @@ def render_source(
                     value=transform.regex,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.REPLACE:
                 key = f"{prefix}-{number}-replace"
@@ -210,8 +235,13 @@ def render_source(
                     value=transform.replace,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.SEPARATOR:
                 key = f"{prefix}-{number}-separator"
@@ -220,8 +250,9 @@ def render_source(
                     value=transform.separator,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             def _handle_remove_transform(field, number):
@@ -230,6 +261,7 @@ def render_source(
             col4.button(
                 "✖️",
                 key=f"{prefix}-{number}-remove-transform",
                 on_click=_handle_remove_transform,
                 args=(field, number),
             )
@@ -243,16 +275,15 @@ def render_source(
     col1.button(
         "Add transform on data",
         key=f"{prefix}-close-fields",
         on_click=_handle_add_transform,
         args=(field,),
     )
 def render_references(
-    record_set_key: int,
     record_set: RecordSet,
     field: Field,
-    field_key: int,
     possible_sources: list[str],
 ):
     """Renders the form for references."""
@@ -286,6 +317,7 @@ def render_references(
                 index=_get_extract_index(references),
                 key=key,
                 options=EXTRACT_TYPES,
                 on_change=handle_field_change,
                 args=(FieldEvent.REFERENCE_EXTRACT, field, key),
             )
@@ -295,6 +327,7 @@ def render_references(
                     needed_field("Column name"),
                     value=references.extract.column,
                     key=key,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_COLUMN, field, key),
                 )
@@ -304,12 +337,14 @@ def render_references(
                     needed_field("JSON path"),
                     value=references.extract.json_path,
                     key=key,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_JSON_PATH, field, key),
                 )
         col4.button(
             "✖️",
             key=f"{key}-remove-reference",
             on_click=_handle_remove_reference,
             args=(field,),
         )

 import mlcroissant as mlc
 from utils import needed_field
+_JSON_PATH_DOCUMENTATION = (
+    "The JSON path if the data source is a JSON (see"
+    " [documentation](https://www.ietf.org/archive/id/draft-goessner-dispatch-jsonpath-00.html))."
+)
+_EXTRACT_DOCUMENTATION = (
+    "The extraction method to get the value of the field (column in a CSV, etc)."
+)
+_COLUMN_NAME_DOCUMENTATION = "The name of the column if the data source is a CSV."
 class SourceType:
     """The type of the source (distribution or field)."""
 def render_source(
     record_set: RecordSet,
     field: Field,
     possible_sources: list[str],
 ):
     """Renders the form for the source."""
         index = None
     key = f"{prefix}-source"
     col1.selectbox(
+        needed_field("Data source"),
         index=index,
         options=options,
         key=key,
+        help=(
+            "Data sources can be other resources (FileObject, FileSet) or other fields."
+        ),
         on_change=handle_field_change,
         args=(FieldEvent.SOURCE, field, key),
     )
             needed_field("Extract"),
             index=_get_extract_index(source),
             key=f"{prefix}-extract",
+            help=_EXTRACT_DOCUMENTATION,
             options=EXTRACT_TYPES,
             on_change=handle_field_change,
             args=(FieldEvent.SOURCE_EXTRACT, field, key),
                 needed_field("Column name"),
                 value=source.extract.column,
                 key=key,
+                help=_COLUMN_NAME_DOCUMENTATION,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_COLUMN, field, key),
             )
                 needed_field("JSON path"),
                 value=source.extract.json_path,
                 key=key,
+                help=_JSON_PATH_DOCUMENTATION,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_JSON_PATH, field, key),
             )
                 key=key,
                 options=TRANSFORM_TYPES,
                 on_change=handle_field_change,
+                help="One or more transformations to apply after extracting the field.",
                 args=(FieldEvent.TRANSFORM, field, key),
                 kwargs={"number": number},
             )
             if selected == TransformType.FORMAT:
                 key = f"{prefix}-{number}-transform-format"
                 col3.text_input(
+                    needed_field("Format a date"),
                     value=transform.format,
                     key=key,
                     on_change=handle_field_change,
+                    help=(
+                        "For dates, use [`Python format"
+                        " codes`](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes)."
+                    ),
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             elif selected == TransformType.JSON_PATH:
                 key = f"{prefix}-{number}-jsonpath"
                     value=transform.json_path,
                     key=key,
                     on_change=handle_field_change,
+                    help=_JSON_PATH_DOCUMENTATION,
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             elif selected == TransformType.REGEX:
                 key = f"{prefix}-{number}-regex"
                     value=transform.regex,
                     key=key,
                     on_change=handle_field_change,
+                    help=(
+                        "A regular expression following [`re` Python"
+                        " convention](https://docs.python.org/3/library/re.html#regular-expression-syntax)"
+                        " with one capturing group. The result of the operation will be"
+                        " the last captured group."
+                    ),
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             elif selected == TransformType.REPLACE:
                 key = f"{prefix}-{number}-replace"
                     value=transform.replace,
                     key=key,
                     on_change=handle_field_change,
+                    help=(
+                        "A replace pattern separated by a `/`, i.e."
+                        " `string_to_replace/string_to_substitute` in order to replace"
+                        " `string_to_replace` by `string_to_substitute`."
+                    ),
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             elif selected == TransformType.SEPARATOR:
                 key = f"{prefix}-{number}-separator"
                     value=transform.separator,
                     key=key,
                     on_change=handle_field_change,
+                    help="A separator to split strings on, e.g. `|` to split `a|b|c`.",
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             def _handle_remove_transform(field, number):
             col4.button(
                 "✖️",
                 key=f"{prefix}-{number}-remove-transform",
+                help="Remove the transformation.",
                 on_click=_handle_remove_transform,
                 args=(field, number),
             )
     col1.button(
         "Add transform on data",
         key=f"{prefix}-close-fields",
+        help="Add a transformation.",
         on_click=_handle_add_transform,
         args=(field,),
     )
 def render_references(
     record_set: RecordSet,
     field: Field,
     possible_sources: list[str],
 ):
     """Renders the form for references."""
                 index=_get_extract_index(references),
                 key=key,
                 options=EXTRACT_TYPES,
+                help=_EXTRACT_DOCUMENTATION,
                 on_change=handle_field_change,
                 args=(FieldEvent.REFERENCE_EXTRACT, field, key),
             )
                     needed_field("Column name"),
                     value=references.extract.column,
                     key=key,
+                    help=_COLUMN_NAME_DOCUMENTATION,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_COLUMN, field, key),
                 )
                     needed_field("JSON path"),
                     value=references.extract.json_path,
                     key=key,
+                    help=_JSON_PATH_DOCUMENTATION,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_JSON_PATH, field, key),
                 )
         col4.button(
             "✖️",
             key=f"{key}-remove-reference",
+            help="Remove the join.",
             on_click=_handle_remove_reference,
             args=(field,),
         )

views/splash.py CHANGED Viewed

@@ -13,6 +13,8 @@ import mlcroissant as mlc
 from views.load import render_load
 from views.previous_files import render_previous_files
 _DATASETS = {
     "Titanic": ["data/embarkation_ports.csv", "data/genders.csv"],
     "FLORES-200": [],
@@ -23,8 +25,23 @@ _DATASETS = {
     "Bigcode-The-Stack": [],
 }
 def render_splash():
     if OAUTH_CLIENT_ID:
         st.info(
             "**Disclaimer**: Do not put sensitive information or datasets here. The"
@@ -34,9 +51,7 @@ def render_splash():
         )
     col1, col2 = st.columns([1, 1], gap="large")
     with col1:
-        with st.expander("**Load an existing Croissant JSON-LD file**", expanded=True):
-            render_load()
-        with st.expander("**Create from scratch**", expanded=True):
             def create_new_croissant():
                 st.session_state[Metadata] = Metadata()
@@ -47,7 +62,7 @@ def render_splash():
                 on_click=create_new_croissant,
                 type="primary",
             )
-        with st.expander("**Try out an example!**", expanded=True):
             def create_example(dataset: str):
                 base = f"https://raw.githubusercontent.com/mlcommons/croissant/main/datasets/{dataset.lower()}"
@@ -72,7 +87,7 @@ def render_splash():
                     )
             dataset = st.selectbox(
-                label="Dataset",
                 options=_DATASETS.keys(),
             )
             st.button(
@@ -81,6 +96,28 @@ def render_splash():
                 type="primary",
                 args=(dataset,),
             )
     with col2:
-        with st.expander("**Past projects**", expanded=True):
             render_previous_files()

 from views.load import render_load
 from views.previous_files import render_previous_files
+_HUGGING_FACE_URL = "https://huggingface.co/datasets/"
 _DATASETS = {
     "Titanic": ["data/embarkation_ports.csv", "data/genders.csv"],
     "FLORES-200": [],
     "Bigcode-The-Stack": [],
 }
+_INFO = """[Croissant](https://mlcommons.org/croissant) 🥐 is a high-level format for
+machine learning datasets built
+on [schema.org](https://schema.org/) and its Dataset vocabulary. A croissant
+configuration file combines metadata, resource file descriptions, data structure, and
+default ML semantics of dataset. You can familiarize yourself with the editor by
+exploring the provided examples.
+The editor supports creating a new configuration from scratch, as well as uploading
+an existing Croissant JSON-MD file. Finally, you can also select any of your
+past projects from the list.
+You can change the project you are currently editing at any time by clicking
+the Menu button and then choosing one of the options on this page."""
 def render_splash():
+    st.info(_INFO, icon="💡")
     if OAUTH_CLIENT_ID:
         st.info(
             "**Disclaimer**: Do not put sensitive information or datasets here. The"
         )
     col1, col2 = st.columns([1, 1], gap="large")
     with col1:
+        with st.expander("**Create a new dataset**", expanded=True):
             def create_new_croissant():
                 st.session_state[Metadata] = Metadata()
                 on_click=create_new_croissant,
                 type="primary",
             )
+        with st.expander("**Load an existing dataset**", expanded=True):
             def create_example(dataset: str):
                 base = f"https://raw.githubusercontent.com/mlcommons/croissant/main/datasets/{dataset.lower()}"
                     )
             dataset = st.selectbox(
+                label="Canonical dataset",
                 options=_DATASETS.keys(),
             )
             st.button(
                 type="primary",
                 args=(dataset,),
             )
+            url = st.text_input(
+                label="Hugging Face dataset",
+                placeholder="Example: https://huggingface.co/datasets/mnist",
+            )
+            if url.startswith(_HUGGING_FACE_URL):
+                name = url.replace(_HUGGING_FACE_URL, "")
+                api_url = (
+                    f"https://datasets-server.huggingface.co/croissant?dataset={name}"
+                )
+                json = requests.get(api_url, headers=None).json()
+                try:
+                    metadata = mlc.Metadata.from_json(mlc.Issues(), json, None)
+                    st.session_state[Metadata] = Metadata.from_canonical(metadata)
+                    save_current_project()
+                except Exception:
+                    st.error(f"Malformed JSON: {json}")
+            elif url:
+                st.error(
+                    f"Unknown URL {url}. Hugging Face URLS should look like"
+                    f" {_HUGGING_FACE_URL}somedataset."
+                )
+            render_load()
     with col2:
+        with st.expander("**Recent projects**", expanded=True):
             render_previous_files()