load json results into a dataset

app.py

@@ -25,10 +25,16 @@ def handle_submit(*inputs):
     if message.startswith("The following fields are required"):
         return message, file_output, json_output
 
+    publish_button = gr.Button(
+        "Share your data to the public repository", interactive=True, elem_classes="pubbutton")
+
+    return "Report sucessefully created", file_output, json_output, publish_button
+
+
+def handle_publi(json_output):
     # If validation passed, proceed to update_dataset
     update_output = update_dataset(json_output)
-    print(json_output)
-    return update_output, file_output, json_output
+    return update_output
 
 
 # Create Gradio interface
@@ -47,11 +53,12 @@ with gr.Blocks(css_paths=css_path) as app:
     quality_components = create_quality_tab()
 
     # Submit and Download Buttons
-    submit_button = gr.Button("Submit")
+    submit_button = gr.Button("Submit", elem_classes="subbutton")
     output = gr.Textbox(label="Output", lines=1)
-    # je comprend pas pq le fichier est vide ???
     json_output = gr.Textbox(visible=False)
     file_output = gr.File(label="Downloadable JSON")
+    publish_button = gr.Button(
+        "Share your data to the public repository", interactive=False, elem_classes="pubbutton")
 
     # Event Handlers
     submit_button.click(
@@ -66,7 +73,15 @@ with gr.Blocks(css_paths=css_path) as app:
             *environment_components,
             *quality_components,
         ],
-        outputs=[output, file_output, json_output]
+        outputs=[output, file_output, json_output, publish_button]
+    )
+    # Event Handlers
+    publish_button.click(
+        handle_publi,
+        inputs=[
+            json_output
+        ],
+        outputs=[output]
     )
 
 if __name__ == "__main__":

assets/styles/app.css

@@ -34,11 +34,25 @@ div {
   color: var(--primary-600);
   font-weight: bold;
 }
+/* for gr.Accordion objects */
+#mandatory_part button > span {
+  color: var(--primary-600);
+  font-weight: bold !important;
+}
 .mandatory_field label > span{
   color: var(--primary-600);
   font-weight: bold !important;
 }
-.mandatory_field div > span{
+.mandatory_field div > span {
   color: var(--primary-600);
   font-weight: bold !important;
 }
+/* submission button */
+.subbutton{
+  background-color: var(--primary-400);
+}
+/* publish button */
+.pubbutton{
+  background-color: #65db86;
+}
+

src/services/huggingface.py

@@ -1,6 +1,7 @@
-import json
 from huggingface_hub import login
-from src.services.util import HF_TOKEN
+from datasets import load_dataset, Dataset, concatenate_datasets
+import json
+from src.services.util import HF_TOKEN, DATASET_NAME
 
 
 def init_huggingface():
@@ -21,10 +22,9 @@ def update_dataset(json_data):
     except json.JSONDecodeError:
         return "Invalid JSON data. Please ensure all required fields are filled correctly."
 
-    # data flattening and saving to dataset
-    """ 
     try:
         dataset = load_dataset(DATASET_NAME, split="train")
+        print(dataset)
     except:
         dataset = Dataset.from_dict({})
 
@@ -32,15 +32,198 @@ def update_dataset(json_data):
     new_dataset = Dataset.from_dict(new_data)
 
     if len(dataset) > 0:
+        print("dataset intitial")
+        print(dataset)
+        print("data to add ")
+        print(new_dataset)
         updated_dataset = concatenate_datasets([dataset, new_dataset])
     else:
         updated_dataset = new_dataset
 
     updated_dataset.push_to_hub(DATASET_NAME)
-    """
     return "Data submitted successfully and dataset updated!"
 
 
+def create_flattened_data(data):
+    """Create a flattened data structure for the algorithms."""
+    # Handle algorithms
+    algorithms = data.get("task", {}).get("algorithms", [])
+    fields = ["trainingType", "algorithmType", "algorithmName", "algorithmUri", "foundationModelName", "foundationModelUri",
+              "parametersNumber", "framework",  "frameworkVersion", "classPath", "layersNumber", "epochsNumber", "optimizer", "quantization"]
+    """Create a flattened data structure for the algorithms."""
+    algorithms_data = {field: "| ".join(str(algo.get(
+        field)) for algo in algorithms if algo.get(field)) or None for field in fields}
+    trainingType_str = algorithms_data["trainingType"]
+    algorithmType_str = algorithms_data["algorithmType"]
+    algorithmName_str = algorithms_data["algorithmName"]
+    algorithmUri_str = algorithms_data["algorithmUri"]
+    foundationModelName_str = algorithms_data["foundationModelName"]
+    foundationModelUri_str = algorithms_data["foundationModelUri"]
+    parametersNumber_str = algorithms_data["parametersNumber"]
+    framework_str = algorithms_data["framework"]
+    frameworkVersion_str = algorithms_data["frameworkVersion"]
+    classPath_str = algorithms_data["classPath"]
+    layersNumber_str = algorithms_data["layersNumber"]
+    epochsNumber_str = algorithms_data["epochsNumber"]
+    optimizer_str = algorithms_data["optimizer"]
+    quantization_str = algorithms_data["quantization"]
+
+    """Create a flattened data structure for the dataset."""
+    # Handle dataset
+    dataset = data.get("task", {}).get("dataset", [])
+    fields = ["dataUsage", "dataType", "dataFormat", "dataSize",
+              "dataQuantity", "shape", "source", "sourceUri",  "owner"]
+    """Create a flattened data structure for the dataset."""
+    dataset_data = {field: "| ".join(
+        str(d.get(field)) for d in dataset if d.get(field)) or None for field in fields}
+    dataUsage_str = dataset_data["dataUsage"]
+    dataType_str = dataset_data["dataType"]
+    dataFormat_str = dataset_data["dataFormat"]
+    dataSize_str = dataset_data["dataSize"]
+    dataQuantity_str = dataset_data["dataQuantity"]
+    shape_str = dataset_data["shape"]
+    source_str = dataset_data["source"]
+    sourceUri_str = dataset_data["sourceUri"]
+    owner_str = dataset_data["owner"]
+
+    """Create a flattened data structure for the measures."""
+    # Handle measures
+    measures = data.get("measures", [])
+    fields = ["measurementMethod", "manufacturer", "version", "cpuTrackingMode", "gpuTrackingMode", "averageUtilizationCpu", "averageUtilizationGpu",
+              "powerCalibrationMeasurement",  "durationCalibrationMeasurement", "powerConsumption", "measurementDuration", "measurementDateTime"]
+    """Create a flattened data structure for the measures."""
+    measures_data = {field: "| ".join(str(measure.get(
+        field)) for measure in measures if measure.get(field)) or None for field in fields}
+    measurementMethod_str = measures_data["measurementMethod"]
+    manufacturer_str = measures_data["manufacturer"]
+    version_str = measures_data["version"]
+    cpuTrackingMode_str = measures_data["cpuTrackingMode"]
+    gpuTrackingMode_str = measures_data["gpuTrackingMode"]
+    averageUtilizationCpu_str = measures_data["averageUtilizationCpu"]
+    averageUtilizationGpu_str = measures_data["averageUtilizationGpu"]
+    powerCalibrationMeasurement_str = measures_data["powerCalibrationMeasurement"]
+    durationCalibrationMeasurement_str = measures_data["durationCalibrationMeasurement"]
+    powerConsumption_str = measures_data["powerConsumption"]
+    measurementDuration_str = measures_data["measurementDuration"]
+    measurementDateTime_str = measures_data["measurementDateTime"]
+
+    # Handle components
+    components = data.get("infrastructure", {}).get("components", [])
+    fields = ["componentName", "componentType", "nbComponent", "memorySize",
+              "manufacturer", "family", "series", "share"]
+
+    # Generate concatenated strings for each field
+    component_data = {field: "| ".join(str(comp.get(
+        field)) for comp in components if comp.get(field)) or None for field in fields}
+
+    componentName_str = component_data["componentName"]
+    componentType_str = component_data["componentType"]
+    nbComponent_str = component_data["nbComponent"]
+    memorySize_str = component_data["memorySize"]
+    manufacturer_infra_str = component_data["manufacturer"]
+    family_str = component_data["family"]
+    series_str = component_data["series"]
+    share_str = component_data["share"]
+
+    return {
+        # Header
+        "licensing": [data.get("header", {}).get("licensing", "")],
+        "formatVersion": [data.get("header", {}).get("formatVersion", "")],
+        "formatVersionSpecificationUri": [data.get("header", {}).get("formatVersionSpecificationUri", "")],
+        "reportId": [data.get("header", {}).get("reportId", "")],
+        "reportDatetime": [data.get("header", {}).get("reportDatetime", "")],
+        "reportStatus": [data.get("header", {}).get("reportStatus", "")],
+        "publisher_name": [data.get("header", {}).get("publisher", {}).get("name", "")],
+        "publisher_division": [data.get("header", {}).get("publisher", {}).get("division", "")],
+        "publisher_projectName": [data.get("header", {}).get("publisher", {}).get("projectName", "")],
+        "publisher_confidentialityLevel": [data.get("header", {}).get("publisher", {}).get("confidentialityLevel", "")],
+        "publisher_publicKey": [data.get("header", {}).get("publisher", {}).get("publicKey", "")],
+
+        # Task
+        "taskStage": [data.get("task", {}).get("taskStage", "")],
+        "taskFamily": [data.get("task", {}).get("taskFamily", "")],
+        "nbRequest": [data.get("task", {}).get("nbRequest", "")],
+        # Algorithms
+        "trainingType": [trainingType_str],
+        "algorithmType": [algorithmType_str],
+        "algorithmName": [algorithmName_str],
+        "algorithmUri": [algorithmUri_str],
+        "foundationModelName": [foundationModelName_str],
+        "foundationModelUri": [foundationModelUri_str],
+        "parametersNumber": [parametersNumber_str],
+        "framework": [framework_str],
+        "frameworkVersion": [frameworkVersion_str],
+        "classPath": [classPath_str],
+        "layersNumber": [layersNumber_str],
+        "epochsNumber": [epochsNumber_str],
+        "optimizer": [optimizer_str],
+        "quantization": [quantization_str],
+        # Dataset
+        "dataUsage": [dataUsage_str],
+        "dataType": [dataType_str],
+        "dataFormat": [dataFormat_str],
+        "dataSize": [dataSize_str],
+        "dataQuantity": [dataQuantity_str],
+        "shape": [shape_str],
+        "source": [source_str],
+        "sourceUri": [sourceUri_str],
+        "owner": [owner_str],
+        "measuredAccuracy": [data.get("task", {}).get("measuredAccuracy", "")],
+        "estimatedAccuracy": [data.get("task", {}).get("estimatedAccuracy", "")],
+        "taskDescription": [data.get("task", {}).get("taskDescription", "")],
+
+        # Measures
+        "measurementMethod": [measurementMethod_str],
+        "manufacturer": [manufacturer_str],
+        "version": [version_str],
+        "cpuTrackingMode": [cpuTrackingMode_str],
+        "gpuTrackingMode": [gpuTrackingMode_str],
+        "averageUtilizationCpu": [averageUtilizationCpu_str],
+        "averageUtilizationGpu": [averageUtilizationGpu_str],
+        "powerCalibrationMeasurement": [powerCalibrationMeasurement_str],
+        "durationCalibrationMeasurement": [durationCalibrationMeasurement_str],
+        "powerConsumption": [powerConsumption_str],
+        "measurementDuration": [measurementDuration_str],
+        "measurementDateTime": [measurementDateTime_str],
+
+        # System
+        "os": [data.get("system", {}).get("os", "")],
+        "distribution": [data.get("system", {}).get("distribution", "")],
+        "distributionVersion": [data.get("system", {}).get("distributionVersion", "")],
+
+        # Software
+        "language": [data.get("software", {}).get("language", "")],
+        "version_software": [data.get("software", {}).get("version_software", "")],
+
+        # Infrastructure
+        "infraType": [data.get("infrastructure", {}).get("infra_type", "")],
+        "cloudProvider": [data.get("infrastructure", {}).get("cloudProvider", "")],
+        "cloudInstance": [data.get("infrastructure", {}).get("cloudInstance", "")],
+        "cloudService": [data.get("infrastructure", {}).get("cloudService", "")],
+        "componentName": [componentName_str],
+        "componentType": [componentType_str],
+        "nbComponent": [nbComponent_str],
+        "memorySize": [memorySize_str],
+        "manufacturer_infra": [manufacturer_infra_str],
+        "family": [family_str],
+        "series": [series_str],
+        "share": [share_str],
+
+        # Environment
+        "country": [data.get("environment", {}).get("country", "")],
+        "latitude": [data.get("environment", {}).get("latitude", "")],
+        "longitude": [data.get("environment", {}).get("longitude", "")],
+        "location": [data.get("environment", {}).get("location", "")],
+        "powerSupplierType": [data.get("environment", {}).get("powerSupplierType", "")],
+        "powerSource": [data.get("environment", {}).get("powerSource", "")],
+        "powerSourceCarbonIntensity": [data.get("environment", {}).get("powerSourceCarbonIntensity", "")],
+
+        # Quality
+        "quality": [data.get("quality", "null")],
+    }
+
+
+""" 
 def create_flattened_data(data):
     out = {}
 
@@ -58,3 +241,4 @@ def create_flattened_data(data):
 
     flatten(data)
     return out
+"""

src/services/util.py

@@ -17,10 +17,20 @@ OBLIGATORY_FIELDS = [
 REPORT_STATUS_OPTIONS = ["draft", "final", "corrective", "other"]
 CONFIDENTIALITY_LEVELS = ["public", "internal", "confidential", "secret"]
 DATA_USAGE_OPTIONS = ["input", "output"]
-DATA_FORMAT = ["3gp", "3gpp", "3gpp2", "8svx", "aa", "aac", "aax", "act", "afdesign", "afphoto", "ai", "aiff", "alac", "amr", "amv", "ape", "arrow", "asf", "au", "avi", "avif", "awb", "bmp", "bpg", "cd5", "cda", "cdr", "cgm", "clip", "cpt", "csv", "deep", "dirac", "divx", "drawingml", "drw", "dss", "dvf", "ecw", "eps", "fits", "flac", "flif", "flv", "flvf4v", "gem", "gerber", "gif", "gle", "gsm", "heif", "hp-gl", "html", "hvif", "ico", "iklax", "ilbm", "img", "ivs", "jpeg", "json", "kra", "lottie", "m4a", "m4b", "m4p", "m4v", "mathml", "matroska", "mdp", "mmf", "movpkg", "mp3", "mpc", "mpeg1",
-               "mpeg2", "mpeg4", "msv", "mxf", "naplps", "netpbm", "nmf", "nrrd", "nsv", "odg", "ods", "ogg", "opus", "pam", "parquet", "pbm", "pcx", "pdf", "pdn", "pgf", "pgm", "pgml", "pict", "plbm", "png", "pnm", "postscript", "ppm", "psd", "psp", "pstricks", "qcc", "quicktime", "ra", "raw", "realmedia", "regis", "rf64", "roq", "sai", "sgi", "sid", "sql", "sln", "svg", "svi", "swf", "text", "tga", "tiff", "tinyvg", "tta", "vicar", "vivoactive", "vml", "vob", "voc", "vox", "wav", "webm", "webp", "wma", "wmf", "wmv", "wv", "xaml", "xar", "xcf", "xisf", "xls", "xlsx", "xml", "xps", "yaml", "other"]
 DATA_TYPES = ["tabular", "audio", "boolean", "image",
               "video", "object", "text", "token", "word", "other"]
+DATA_FORMAT = ["3gp", "3gpp", "3gpp2", "8svx", "aa", "aac", "aax", "act", "afdesign", "afphoto", "ai", "aiff", "alac",
+               "amr", "amv", "ape", "arrow", "asf", "au", "avi", "avif", "awb", "bmp", "bpg", "cd5", "cda", "cdr",
+               "cgm", "clip", "cpt", "csv", "deep", "dirac", "divx", "drawingml", "drw", "dss", "dvf", "ecw", "eps",
+               "fits", "flac", "flif", "flv", "flvf4v", "gem", "gerber", "gif", "gle", "gsm", "heif", "hp-gl", "html", "hvif",
+               "ico", "iklax", "ilbm", "img", "ivs", "jpeg", "json", "kra", "lottie", "m4a", "m4b", "m4p", "m4v", "mathml",
+               "matroska", "mdp", "mmf", "movpkg", "mp3", "mpc", "mpeg1", "mpeg2", "mpeg4", "msv", "mxf", "naplps", "netpbm",
+               "nmf", "nrrd", "nsv", "odg", "ods", "ogg", "opus", "pam", "parquet", "pbm", "pcx", "pdf", "pdn", "pgf", "pgm",
+               "pgml", "pict", "plbm", "png", "pnm", "postscript", "ppm", "psd", "psp", "pstricks", "qcc", "quicktime", "ra",
+               "raw", "realmedia", "regis", "rf64", "roq", "sai", "sgi", "sid", "sql", "sln", "svg", "svi", "swf", "text", "tga",
+               "tiff", "tinyvg", "tta", "vicar", "vivoactive", "vml", "vob", "voc", "vox", "wav", "webm", "webp", "wma", "wmf",
+               "wmv", "wv", "xaml", "xar", "xcf", "xisf", "xls", "xlsx", "xml", "xps", "yaml", "other"]
+
 DATA_SOURCE = ["public", "private", "other"]
 ACCURACY_LEVELS = ["veryPoor", "poor", "average", "good", "veryGood"]
 MEASUREMENT_UNITS = ["Wh", "kWh", "MWh", "GWh", "kJoule", "MJoule", "GJoule", "TJoule", "PJoule",
@@ -31,7 +41,3 @@ POWER_SUPPLIER_TYPES = ["public", "private", "internal", "other"]
 POWER_SOURCES = ["solar", "wind", "nuclear",
                  "hydroelectric", "gas", "coal", "other"]
 QUALITY_LEVELS = ["high", "medium", "low"]
-HASH_ALGORITHMS = ["MD5", "RIPEMD-128", "RIPEMD-160", "RIPEMD-256", "RIPEMD-320",
-                   "SHA-1", "SHA-224", "SHA256", "SHA-384", "SHA-512"]
-CRYPTO_ALGORITHMS = ["RSA", "DSA", "ECDSA", "EDDSA"]
-CACHE_OPTIONS = ["true", "false"]

src/ui/form_components.py

@@ -3,8 +3,7 @@ from src.services.util import (
     REPORT_STATUS_OPTIONS, CONFIDENTIALITY_LEVELS, DATA_USAGE_OPTIONS, DATA_FORMAT,
     DATA_TYPES, DATA_SOURCE,
     ACCURACY_LEVELS, INFRA_TYPES,
-    POWER_SUPPLIER_TYPES, POWER_SOURCES, QUALITY_LEVELS,
-    HASH_ALGORITHMS, CRYPTO_ALGORITHMS
+    POWER_SUPPLIER_TYPES, POWER_SOURCES, QUALITY_LEVELS
 )
 
 
@@ -144,7 +143,7 @@ def create_header_tab():
         reportId = gr.Textbox(
             label="Report ID", info="(the unique identifier of this report, preferably as a uuid4 string)")
         reportDatetime = gr.Textbox(
-            label="Report Datetime", info="(Required field<br>the publishing date of this report in format YYYY-MM-DD HH:MM:SS)", elem_classes="mandatory_field")
+            label="Report Datetime", info="Required field<br>(the publishing date of this report in format YYYY-MM-DD HH:MM:SS)", elem_classes="mandatory_field")
         reportStatus = gr.Dropdown(value=None,
                                    label="Report Status",
                                    choices=REPORT_STATUS_OPTIONS,
@@ -185,9 +184,9 @@ def create_task_tab():
             label="Number of Requests", info="(if inference stage, the number of requests the measure corresponds to, 0 or empty if you're not measuring the inference stage)",
             value=lambda: None, minimum=0)
 
-        with gr.Accordion("Algorithms"):
+        with gr.Accordion("Algorithms", elem_id="mandatory_part"):
             _, trainingType, algorithmType, algorithmName, algorithmUri, foundationModelName, foundationModelUri, parametersNumber, framework,  frameworkVersion, classPath, layersNumber, epochsNumber, optimizer, quantization, add_algorithm_btn = create_dynamic_section(
-                section_name="Algorithms",
+                section_name="algorithm",
                 fields_config=[
                     {
                         "type": gr.Textbox,
@@ -202,7 +201,7 @@ def create_task_tab():
                     {
                         "type": gr.Textbox,
                         "label": "Algorithm Name",
-                        "info": "(the case-sensitive common name of the algorithm, example: randomForest, svm, xgboost...)",
+                        "info": "(the case-sensitive common name of the algorithm, example: randomForest, naive bayes, cnn, rnn, transformers, if you are directly using a foundation model, let it empty and fill the field foundationModelName...)",
                     },
                     {
                         "type": gr.Textbox,
@@ -264,9 +263,9 @@ def create_task_tab():
                 layout="column"
             )
 
-        with gr.Accordion("Dataset"):
+        with gr.Accordion("Dataset", elem_id="mandatory_part"):
             _, dataUsage, dataType, dataFormat, dataSize, dataQuantity, shape, source, sourceUri,  owner, add_dataset_btn = create_dynamic_section(
-                section_name="Dataset",
+                section_name="dataset",
                 fields_config=[
                     {
                         "type": gr.Dropdown,
@@ -353,12 +352,12 @@ def create_measures_tab():
     with gr.Tab("Measures", elem_id="mandatory_part"):
         with gr.Accordion("Measures"):
             _, measurementMethod, manufacturer, version, cpuTrackingMode, gpuTrackingMode, averageUtilizationCpu, averageUtilizationGpu, powerCalibrationMeasurement,  durationCalibrationMeasurement, powerConsumption, measurementDuration, measurementDateTime, add_measurement_btn = create_dynamic_section(
-                section_name="Measures",
+                section_name="measure",
                 fields_config=[
                     {
                         "type": gr.Textbox,
                         "label": "Method of measurement",
-                        "info": "Required field<br>(the energy measure obtained from software and/or hardware tools, for a computing task)",
+                        "info": "Required field<br>(the method used to perform the energy measure, example: codecarbon, carbonai, flops-compute, wattmeter...)",
                         "elem_classes": "mandatory_field",
                     },
                     {
@@ -451,9 +450,9 @@ def create_software_tab():
     """Create the software tab components."""
     with gr.Tab("Software"):
         language = gr.Textbox(
-            label="Language", info="Required field<br>(programming language information)", elem_classes="mandatory_field")
+            label="Language", info="Required field<br>(name of the programming language used, example : c, java, julia, python...)", elem_classes="mandatory_field")
         version_software = gr.Textbox(
-            label="Version", info="(version of the programming language)")
+            label="Version", info="(version of the programming language used)")
 
         return [language, version_software]
 
@@ -473,9 +472,9 @@ def create_infrastructure_tab():
             label="Cloud Instance", info="(If you are on a cloud vm, the name of your cloud instance, for example : a1.large, dasv4-type2...)")
         cloudService = gr.Textbox(
             label="Cloud Service", info="(If you are using an AI cloud service, the name of your cloud service, for example : openAI service...)")
-        with gr.Accordion("Components"):
+        with gr.Accordion("Components", elem_id="mandatory_part"):
             _, componentName, componentType, nbComponent, memorySize, manufacturer_infra, family, series, share, add_component_btn = create_dynamic_section(
-                section_name="Component",
+                section_name="component",
                 fields_config=[
 
                     {
@@ -566,7 +565,7 @@ def create_quality_tab():
         quality = gr.Dropdown(value=None,
                               label="Quality",
                               choices=QUALITY_LEVELS,
-                              info="(the quality of the information provided)"
+                              info="(the quality of the information you provided, 3 possibilities : high (percentage error +/-10%), medium (percentage error +/-25%), low (percentage error +/-50%))"
                               )
 
         return [quality]