Preprocessing

app.py

@@ -15,91 +15,21 @@ HfFolder.save_token(HF_TOKEN)
 
 
 datasets = {
-    "stars": load_dataset("open-source-metrics/stars").sort('dates'),
-    "issues": load_dataset("open-source-metrics/issues").sort('dates'),
+    "stars": load_dataset("open-source-metrics/preprocessed_stars"),
+    "issues": load_dataset("open-source-metrics/preprocessed_issues"),
     "pip": load_dataset("open-source-metrics/pip").sort('day'),
 }
 
 external_datasets = {
-    "stars": load_dataset("open-source-metrics/stars-external").sort('dates'),
-    "issues": load_dataset("open-source-metrics/issues-external").sort('dates'),
     "pip": load_dataset("open-source-metrics/pip-external").sort('day')
 }
 
-val = 0
 
-
-def _range(e):
-    global val
-    e['range'] = val
-    val += 1
-
-    current_date = datetime.strptime(e['dates'], "%Y-%m-%dT%H:%M:%SZ")
-    first_date = datetime.fromtimestamp(1)
-    week = abs(current_date - first_date).days // 7
-    e['week'] = week
-
-    return e
-
-
-def _ignore_org_members(e):
-    global val
-    e['range_non_org'] = val
-
-    if e['type']['authorAssociation'] != 'MEMBER':
-        val += 1
-
-    return e
-
-
-stars = {}
-for k, v in datasets['stars'].items():
-    stars[k] = v.map(_range)
-    val = 0
-
-stars_external = {}
-for k, v in external_datasets['stars'].items():
-    stars_external[k] = v.map(_range)
-    val = 0
-
-
-issues = {}
-for k, v in datasets['issues'].items():
-    issues[k] = v.map(_range)
-    val = 0
-    issues[k] = issues[k].map(_ignore_org_members)
-    val = 0
-
-
-issues_external = {}
-for k, v in external_datasets['issues'].items():
-    issues_external[k] = v.map(_range)
-    val = 0
-    issues_external[k] = issues_external[k].map(_ignore_org_members)
-    val = 0
-
-datasets['stars'] = DatasetDict(**stars)
-datasets['issues'] = DatasetDict(**issues)
-external_datasets['stars'] = DatasetDict(**stars_external)
-external_datasets['issues'] = DatasetDict(**issues_external)
-
-
-def link_values(library_names, returned_values):
-    previous_values = {library_name: None for library_name in library_names}
-    for library_name in library_names:
-        for i in returned_values.keys():
-            if library_name not in returned_values[i]:
-                returned_values[i][library_name] = previous_values[library_name]
-            else:
-                previous_values[library_name] = returned_values[i][library_name]
-
-    return returned_values
-
-
-def running_mean(x, N, total_length=-1):
-    cumsum = np.cumsum(np.insert(x, 0, 0))
-    to_pad = max(total_length - len(cumsum), 0)
-    return np.pad(cumsum[N:] - cumsum[:-N], (to_pad, 0)) / float(N)
+def cut_output(full_output: Dataset, library_names: list):
+    output = full_output.to_dict().items()
+    output = {k: v + [None] for k, v in output if k in library_names + ['day']}
+    last_value = max(output[k].index(None) for k in output.keys() if k != 'day')
+    return {k: v[:last_value] for k, v in output.items()}
 
 
 def parse_name_and_options(path):
@@ -152,10 +82,12 @@ class RequestHandler(SimpleHTTPRequestHandler):
             external_dataset_with_most_splits = list(external_dataset_with_most_splits)
             external_dataset_with_most_splits.sort()
 
+            warnings.append("Selecting PyTorch and/or TensorFlow will take a while to compute, and may timeout for issues/PRs..")
+
             res = {
                 'internal': dataset_with_most_splits,
                 'external': external_dataset_with_most_splits,
-                'warnings': warnings
+                'warnings': []
             }
 
             print(f"Returning: {res}")
@@ -215,90 +147,29 @@ class RequestHandler(SimpleHTTPRequestHandler):
             return self.response(output)
 
         if self.path.startswith("/retrieveStars"):
-            errors = []
             library_names, options = parse_name_and_options(self.path)
-            returned_values = {}
-            dataset_dict = datasets['stars']
-            external_dataset_dict = external_datasets['stars']
             week_over_week = '1' in options
 
-            for library_name in library_names:
-                if library_name in dataset_dict:
-                    dataset = dataset_dict[library_name]
-                elif library_name in external_dataset_dict:
-                    dataset = external_dataset_dict[library_name]
-                else:
-                    errors.append(f"No {library_name} found in internal or external datasets for stars.")
-                    return {'errors': errors}
-
-                last_value = 0
-                last_week = dataset[0]['week']
-                for i in dataset:
-                    if week_over_week and last_week == i['week']:
-                        continue
-                    if i['dates'] in returned_values:
-                        returned_values[i['dates']][library_name] = i['range'] - last_value
-                    else:
-                        returned_values[i['dates']] = {library_name: i['range'] - last_value}
-
-                    last_value = i['range'] if week_over_week else 0
-                    last_week = i['week']
-
-            returned_values = collections.OrderedDict(sorted(returned_values.items()))
-            returned_values = link_values(library_names, returned_values)
-            output = {l: [k[l] for k in returned_values.values()][::-1] for l in library_names}
-            output['day'] = list(returned_values.keys())[::-1]
-
-            # Trim down to a smaller number of points.
-            output = {k: [v for i, v in enumerate(value) if i % max(1, int(len(value) / 100)) == 0] for k, value in output.items()}
-
-            return self.response(output)
-
+            if week_over_week:
+                return self.response({k: v for k, v in datasets['stars']['wow'].to_dict().items() if k in library_names + ['day']})
+            else:
+                return self.response({k: v for k, v in datasets['stars']['wow'].to_dict().items() if k in library_names + ['day']})
 
         if self.path.startswith("/retrieveIssues"):
-            errors = []
             library_names, options = parse_name_and_options(self.path)
-
             exclude_org_members = '1' in options
             week_over_week = '2' in options
 
-            returned_values = {}
-            dataset_dict = datasets['issues']
-            external_dataset_dict = external_datasets['issues']
-            range_id = 'range' if not exclude_org_members else 'range_non_org'
-
-            for library_name in library_names:
-                if library_name in dataset_dict:
-                    dataset = dataset_dict[library_name]
-                elif library_name in external_dataset_dict:
-                    dataset = external_dataset_dict[library_name]
+            if week_over_week:
+                if exclude_org_members:
+                    return self.response(cut_output(datasets['issues']['eom_wow'], library_names))
                 else:
-                    errors.append(f"No {library_name} found in internal or external datasets for stars.")
-                    return {'errors': errors}
-
-                last_value = 0
-                last_week = dataset[0]['week']
-                for i in dataset:
-                    if week_over_week and last_week == i['week']:
-                        continue
-
-                    if i['dates'] in returned_values:
-                        returned_values[i['dates']][library_name] = i[range_id] - last_value
-                    else:
-                        returned_values[i['dates']] = {library_name: i[range_id] - last_value}
-
-                    last_value = i[range_id] if week_over_week else 0
-                    last_week = i['week']
-
-            returned_values = collections.OrderedDict(sorted(returned_values.items()))
-            returned_values = link_values(library_names, returned_values)
-            output = {l: [k[l] for k in returned_values.values()][::-1] for l in library_names}
-            output['day'] = list(returned_values.keys())[::-1]
-
-            # Trim down to a smaller number of points.
-            output = {k: [v for i, v in enumerate(value) if i % max(1, int(len(value) / 100)) == 0] for k, value in output.items()}
-
-            return self.response(output)
+                    return self.response({k: v for k, v in datasets['issues']['wow'].to_dict().items() if k in library_names + ['day']})
+            else:
+                if exclude_org_members:
+                    return self.response({k: v for k, v in datasets['issues']['eom'].to_dict().items() if k in library_names + ['day']})
+                else:
+                    return self.response({k: v for k, v in datasets['issues']['raw'].to_dict().items() if k in library_names + ['day']})
 
         return SimpleHTTPRequestHandler.do_GET(self)
 

index.js

@@ -122,16 +122,16 @@ const initialize = async () => {
     graphSelector.appendChild(graphSpan);
 
     if (inferJson.warnings.length > 0) {
-        const div = document.createElement('div');
-        div.classList.add('warning-div')
-
         for (const warning of inferJson.warnings) {
+            const div = document.createElement('div');
+            div.classList.add('warning-div')
+
             const labelSpan = document.createElement('span');
             labelSpan.textContent = `Warning: ${warning}`;
 
             div.appendChild(labelSpan);
+            warnings.appendChild(div);
         }
-        warnings.appendChild(div);
     }
 
     for (const element of inferJson.internal) {