Reviewed for comments and type hints

src/data_synthesis/select_test_data_from_all_products.py

@@ -32,6 +32,9 @@ def dst_conn() -> sqlite3.Connection:
 
 
 def copy_all_products_db() -> None:
+    """
+    Copies the source to the target database via a simple file copy.
+    """
     shutil.copy(src_db(), dst_db())
 
 
@@ -42,7 +45,10 @@ def join_strings_for_sql(items: List[str]) -> str:
     return ', '.join([f'"{i}"' for i in items])
 
 
-def execute_sqls(sql_statements: List[str]):
+def execute_sqls(sql_statements: List[str]) -> None:
+    """
+    Executes a number of sql statements against the destination DB
+    """
     db_connection = dst_conn()
     db_cursor = db_connection.cursor()
     for s in sql_statements:
@@ -51,6 +57,10 @@ def execute_sqls(sql_statements: List[str]):
 
 
 def execute_select(sql_statement: str) -> List[List]:
+    """
+    Executes a single sql statement against the destination DB and returns the
+    complete associated resultset
+    """
     db_connection = dst_conn()
     db_cursor = db_connection.cursor()
     db_cursor.execute(sql_statement)
@@ -58,7 +68,10 @@ def execute_select(sql_statement: str) -> List[List]:
 
 
 def drop_categories(keeping: List[str]) -> None:
-    # Drop from the tables sequentially to avoid foreign key constraint issues
+    """
+    Drops whole categories from the destination DB. Only keeps the ones named in the list.
+    Drops from tables seuqentially to respect foreign key constraints.
+    """
     keep_in = f"({join_strings_for_sql(keeping)})"
     sql_statements = []
     sql = f"delete from reviews where product_id in (select id from products where category_id in (select id from categories where name not in {keep_in}))"
@@ -76,6 +89,11 @@ def drop_categories(keeping: List[str]) -> None:
 
 
 def drop_products(ids: List[int]) -> None:
+    """
+    Drops a number of products from the destination DB based on IDs. Drops from tables sequentially to respect foreign
+    key constraints
+    :param ids: the IDs of the products to drop
+    """
     ids_in = f"({join_strings_for_sql(ids)})"
 
     sql_statements = []
@@ -88,7 +106,10 @@ def drop_products(ids: List[int]) -> None:
     execute_sqls(sql_statements)
 
 
-def winnow_prods_per_category(min_count: int = 10, max_count: int = 25):
+def winnow_prods_per_category(min_count: int = 10, max_count: int = 25) -> None:
+    """
+    Thins out the number of products in each category so the number falls somewhere (randomly) within a target range
+    """
     sql = "select name from categories"
     categories = [r[0] for r in execute_select(sql)]
     for c in categories:
@@ -105,12 +126,22 @@ def winnow_prods_per_category(min_count: int = 10, max_count: int = 25):
 
 
 def avg_rating(review_ratings1: List[List[int]], review_ratings2: List[List[int]]) -> float:
+    """
+    Calculates the average rating across two lists of reviews
+    """
     review_count = len(review_ratings1) + len(review_ratings2)
     review_sum = sum([r[1] for r in review_ratings1]) + sum([r[1] for r in review_ratings2])
     return review_sum / review_count
 
 
 def get_review_ids_to_drop(review_ratings: List[List[int]], target_review_count: int, target_avg_rating: float) -> List[int]:
+    """
+    Selects a set of reviews to drop from the dataset whilst trying to meet two objectives.  First is to get close to
+    a target average review value (i.e. review stars for this product).  Second is to do this while arriving at a
+    requested number of overall reviews. The approach is to split the reviews to higher than target / lower than target
+    and then repeatedly randomly select either a higher than target or lower than target individual review to
+    move the average in the right direction and get an approximately good solution.
+    """
     ids_to_drop = []
     rated_lower_than_target = [r for r in review_ratings if r[1] <= target_avg_rating]
     rated_higher_than_target = [r for r in review_ratings if r[1] > target_avg_rating]
@@ -128,7 +159,10 @@ def get_review_ids_to_drop(review_ratings: List[List[int]], target_review_count:
     return ids_to_drop
 
 
-def drop_reviews(review_ids: list[int]):
+def drop_reviews(review_ids: list[int]) -> None:
+    """
+    Drops specific review IDs from the destination DB.
+    """
     if len(review_ids) == 0:
         print("Warning - got zero reviews to drop")
     else:
@@ -141,6 +175,10 @@ def drop_reviews_to_balance_avg_rating(min_review_count: int = 5,
                                        max_review_count: int = 30,
                                        min_target_avg_rating: float = 3.4,
                                        max_target_avg_rating: float = 4.9):
+    """
+    For each product generates a target rating in a range and a target number of reviews in a range
+    and then runs a process to try and get close to a solution for that product.
+    """
     sql = "select id, name from products"
     product_id_names = [(r[0], r[1]) for r in execute_select(sql)]
     for prod_id, prod_name in product_id_names: