Create utils.py

utils.py

@@ -0,0 +1,148 @@
+import numpy as np
+import pandas as pd
+
+from sklearn.model_selection import ShuffleSplit, train_test_split, GridSearchCV
+from sklearn.neighbors import KNeighborsRegressor, NearestNeighbors
+from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error
+
+
+SEED = 8642
+np.random.seed(SEED)
+
+config = {'min_xval':55.55, 'max_xval':55.95, 'min_yval':37.3, 'max_yval':37.9, 'x_ngroups': 8, 'y_ngroups': 8}
+
+def split_on_intervals(min_val, max_val, n):
+    step = (max_val - min_val)/n
+    intervals = [min_val+(step*x) for x in range(n+1)]
+    return intervals
+
+
+def create_groups(x_intervals, y_intervals):
+    groups = {}
+    x_intervals = np.concatenate([[-np.inf], x_intervals, [np.inf]])
+    y_intervals = np.concatenate([[-np.inf], y_intervals, [np.inf]])
+    for x_i in range(len(x_intervals)-1):
+        for y_i in range(len(y_intervals)-1):
+            groups[f'x : {x_intervals[x_i]} - {x_intervals[x_i+1]} | y : {y_intervals[y_i]} - {y_intervals[y_i+1]}'] = 0
+    return groups
+
+
+#Сортирует точки по регионам
+def sort_on_groups(x_vals, y_vals, x_intervals, y_intervals, groups, only_vals = False):
+    for x, y in zip(x_vals, y_vals):
+        for x_i in range(len(x_intervals)-1):
+            for y_i in range(len(y_intervals)-1):
+                if ((x_intervals[x_i] <= x < x_intervals[x_i+1]) and (y_intervals[y_i] <= y < y_intervals[y_i+1])):
+                    groups[f'x : {x_intervals[x_i]} - {x_intervals[x_i+1]} | y : {y_intervals[y_i]} - {y_intervals[y_i+1]}'] += 1
+
+    if only_vals:
+        return list(groups.values())
+
+    return groups
+
+
+def create_dataset(config, df):
+    x_intervals = split_on_intervals(config['min_xval'], config['max_xval'], config['x_ngroups'])
+    y_intervals = split_on_intervals(config['min_yval'], config['max_yval'], config['y_ngroups'])
+    groups = create_groups(x_intervals, y_intervals)
+
+    groups_values = []
+    for i in range(len(df)):
+        g = df.iloc[i]
+        points = np.array([[float(x['lat']), float(x['lon'])] for x in g['points']])
+        group_values = sort_on_groups(points[:,0], points[:,1], x_intervals, y_intervals, groups.copy(), only_vals = True)
+        groups_values.append(group_values)
+
+    groups_values = np.array(groups_values)
+    for i in range(len(groups.keys())):
+        groups[list(groups.keys())[i]]=groups_values[:,i]
+
+    return groups
+
+
+def feature_engineering(df: pd.DataFrame, n_groups: int = 8, parse_json: bool = True, dists: bool = True, 
+                        age_feats: bool = True, normalize: bool = True, drop_zero_cols: bool = False,
+                        process_points: bool = True) -> pd.DataFrame:
+
+    df.reset_index(drop=True, inplace=True)
+
+    if parse_json:
+        df_norm = pd.concat([df, pd.json_normalize(df['targetAudience'])], axis=1)
+        # print(df_norm)
+        df_norm = df_norm.drop(columns=['targetAudience'])
+    else:
+        df_norm = df
+
+    if process_points:
+        df_norm['points_count'] = df_norm['points'].apply(len)
+        if dists:
+            lat_center = 55.7522
+            lon_center = 37.6156
+            # print(df_norm)
+            df_norm['distance_center'] = df_norm['points'].apply(lambda points: [((float(point['lat']) - lat_center) ** 2 + (float(point['lon']) - lon_center) ** 2) ** 0.5 for point in points]).apply(sum) / df_norm['points_count']
+            lat_patr = 55.763868
+            lon_patr = 37.592168
+            df_norm['distance_patriki'] = df_norm['points'].apply(lambda points: [((float(point['lat']) - lat_patr) ** 2 + (float(point['lon']) - lon_patr) ** 2) ** 0.5 for point in points]).apply(sum) / df_norm['points_count']
+            lat_luzh = 55.717934
+            lon_luzh = 37.551932
+            df_norm['distance_luzhniki'] = df_norm['points'].apply(lambda points: [((float(point['lat']) - lat_luzh) ** 2 + (float(point['lon']) - lon_luzh) ** 2) ** 0.5 for point in points]).apply(sum) / df_norm['points_count']
+
+        config = {'min_xval': 55.55, 'max_xval': 55.95, 'min_yval': 37.3, 'max_yval': 37.9, 'x_ngroups': 8, 'y_ngroups': 8}
+        dataset = pd.DataFrame(create_dataset(config, df_norm))
+        if drop_zero_cols:
+            zero_columns = dataset.sum()[dataset.sum() == 0].index.tolist()
+            dataset = dataset.drop(columns=zero_columns)
+    else:
+        dataset = pd.DataFrame()
+
+    df_new = df_norm
+    for col in ['hash', 'points', 'name']:
+        try:
+            df_new = df_new.drop(columns=[col])
+        except: pass
+
+    if age_feats:
+        df_new['age_span'] = df_new.apply(lambda row: (row['ageTo'] - row['ageFrom']) ** 1/2, axis=1)
+        df_new['age_mean'] = df_new.apply(lambda row: ((row['ageTo'] + row['ageFrom']) / 2) ** 1/2, axis=1)
+
+    gender_ohe = pd.get_dummies(df_new['gender'])
+    try: 
+        income_ohe = pd.get_dummies(df_new['income']).drop(columns=['ac'])
+    except:
+        print('no ac in income')
+        income_ohe = pd.get_dummies(df_new['income'])
+    
+    df_full = pd.concat([dataset, df_new, gender_ohe, income_ohe], axis=1).drop(columns=['gender', 'income'])
+    df_full = df_full.fillna(0)
+
+    if normalize:
+        df_scaled = (df_full - df_full.mean()) / df_full.std()
+        df_scaled = df_scaled.fillna(0)
+        return df_scaled
+    else:
+        return df_full
+
+
+def fit_to_base_cols(df_scaled: pd.DataFrame, base_cols) -> pd.DataFrame:
+    # base_df = pd.DataFrame(columns=base_cols)
+    # for col in df_scaled.columns:
+    #     base_df[col] = df_scaled[col]
+    # base_df = base_df.fillna(0)
+    missing_cols = list(set(base_cols.tolist()) - set(df_scaled.columns.tolist()))
+    for col in missing_cols:
+        df_scaled[col] = 0
+    return df_scaled
+
+
+def prep_test_df(test_df: pd.DataFrame, base_cols) -> pd.DataFrame:
+    sc_df = feature_engineering(test_df, parse_json=True, normalize=False, drop_zero_cols=False)
+    sc_df = fit_to_base_cols(sc_df, base_cols)
+    return sc_df
+
+
+def ensemble_inference(models, X_test):
+    y_pred = np.zeros(len(X_test))
+    for model in models[:]:
+        y_pred += model.predict(X_test)
+    y_pred /= len(models)
+    return y_pred