Switch e-commerce platform

app.py

@@ -1,141 +1,125 @@
-import re
-import requests
-import pandas as pd
-import unicodedata as uni
-import emoji
-from langchain.chat_models import ChatOpenAI
-from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.document_loaders import DataFrameLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores import FAISS
-from langchain.chains import RetrievalQA
-import gradio as gr
-
-SHOPEE_API_URL = """https://shopee.co.id/api/v2/item/get_ratings?filter=0&flag=1&itemid={item_id}&limit=20&offset={offset}&shopid={shop_id}&type=0"""
-shop_id = ""
-item_id = ""
-item = {}
-LIMIT = 1000  # Limit to 1000 reviews so that processing does not take too long
-
-
-def get_product_id(URL):
-    # Get shop id and item id from input URL
-    r = re.search(r"i\.(\d+)\.(\d+)", URL)
-    shop_id, item_id = r[1], r[2]
-    return shop_id, item_id
-
-
-def scrape(URL):
-    try:
-        shop_id, item_id = get_product_id(URL)
-    except:
-        return None
-
-    offset = 0
-    reviews = []
-    while True:
-        # Get JSON data using shop_id and item_id from input URL
-        data = requests.get(
-            SHOPEE_API_URL.format(shop_id=shop_id, item_id=item_id, offset=offset)
-        ).json()
-
-        i = 1
-        for i, review in enumerate(data["data"]["ratings"], 1):
-            reviews.append(review["comment"])
-
-        if i % 20:
-            break
-
-        offset += 20
-        if offset >= LIMIT:
-            break
-
-    df = pd.DataFrame(reviews, columns=["comment"])
-
-    return df
-
-
-# Clean
-def clean(df):
-    df = df.dropna().copy().reset_index(drop=True)  # drop reviews with empty comments
-    df = df[df["comment"] != ""].reset_index(drop=True)  # remove empty reviews
-    df["comment"] = df["comment"].apply(lambda x: clean_text(x))  # clean text
-    df = df[df["comment"] != ""].reset_index(drop=True)  # remove empty reviews
-    return df
-
-
-def clean_text(text):
-    text = uni.normalize("NFKD", text)  # normalise characters
-    text = emoji.replace_emoji(text, "")  # remove emoji
-    text = re.sub(r"(\w)\1{2,}", r"\1", text)  # repeated chars
-    text = re.sub(r"[ ]+", " ", text).strip()  # remove extra spaces
-    return text
-
-
-# LLM
-OpenAIModel = "gpt-3.5-turbo"
-llm = ChatOpenAI(model=OpenAIModel, temperature=0.1)
-
-# Embeddings
-embeddings = HuggingFaceEmbeddings(model_name="Blaxzter/LaBSE-sentence-embeddings")
-
-cache_URL = ""
-db = None
-qa = None
-
-
-def generate(URL, query):
-    global cache_URL, db, qa
-    if URL != cache_URL:
-        # Get reviews
-        try:
-            reviews = scrape(URL)
-            # Clean reviews
-            cleaned_reviews = clean(reviews)
-            # Load data
-            loader = DataFrameLoader(cleaned_reviews, page_content_column="comment")
-            documents = loader.load()
-        except Exception as e:
-            return "Error getting reviews: " + str(e)
-
-        # Split text
-        text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=1000, chunk_overlap=50
-        )
-        docs = text_splitter.split_documents(documents)
-        cache_URL = URL
-        # Vector store
-        db = FAISS.from_documents(docs, embeddings)
-        # Chain to answer questions
-        qa = RetrievalQA.from_chain_type(llm=llm, retriever=db.as_retriever())
-    return qa.run(query)
-
-
-# Gradio
-product_box = gr.Textbox(
-    label="URL Produk", placeholder="URL produk dari Shopee Indonesia"
-)
-query_box = gr.Textbox(
-    lines=2,
-    label="Kueri",
-    placeholder="Contoh: Apa yang orang katakan tentang kualitas produknya?, Bagaimana pendapat orang yang kurang puas dengan produknya?",
-)
-
-gr.Interface(
-    fn=generate,
-    inputs=[product_box, query_box],
-    outputs=gr.Textbox(label="Jawaban"),
-    title="RingkasUlas",
-    description="Bot percakapan yang bisa meringkas ulasan-ulasan produk di Shopee Indonesia (https://shopee.co.id/). Harap bersabar, bot ini dapat memakan waktu agak lama saat mengambil ulasan dari Shopee dan menyiapkan jawabannya.",
-    allow_flagging="never",
-    examples=[
-        [
-            "https://shopee.co.id/Bantal-Selimut-Balmut-Mini-Karakter-kain-CVC-i.2392232.8965506?xptdk=324a77c0-7860-4059-b00d-5d3b340f8dfe",
-            "Apa yang orang katakan tentang kualitas produknya?",
-        ],
-        [
-            "https://shopee.co.id/Bantal-Selimut-Balmut-Mini-Karakter-kain-CVC-i.2392232.8965506?xptdk=324a77c0-7860-4059-b00d-5d3b340f8dfe",
-            "Bagaimana pendapat orang yang kurang puas dengan produknya?",
-        ],
-    ],
-).launch()
+!pip install transformers sentence_transformers langchain openai python-dotenv chromadb faiss-gpu
+import re
+from urllib.parse import urlparse, parse_qs
+import pandas as pd
+import unicodedata as uni
+import emoji
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.document_loaders import DataFrameLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import FAISS
+from langchain.chains import RetrievalQA
+from tokopedia import request_product_id, request_product_review
+import gradio as gr
+
+shop_id = ""
+item_id = ""
+item = {}
+LIMIT = 1000  # Limit to 1000 reviews so that processing does not take too long
+
+def scrape(URL, max_reviews=LIMIT):
+    parsed_url = urlparse(URL)
+    *_, SHOP, PRODUCT_KEY = parsed_url.path.split("/")
+    product_id = request_product_id(SHOP, PRODUCT_KEY).json()["data"]["pdpGetLayout"][
+        "basicInfo"
+    ]["id"]
+    all_reviews = []
+    page = 1
+    has_next = True
+
+    while has_next and len(all_reviews) <= max_reviews:
+        response = request_product_review(product_id, page=page)
+        data = response.json()["data"]["productrevGetProductReviewList"]
+        reviews = data["list"]
+        all_reviews.extend(reviews)
+        has_next = data["hasNext"]
+        page += 1
+
+    reviews_df = pd.json_normalize(all_reviews)
+    return reviews_df
+
+# Clean
+def clean(df):
+    df = df.dropna().copy().reset_index(drop=True)  # drop reviews with empty comments
+    df = df[df["comment"] != ""].reset_index(drop=True)  # remove empty reviews
+    df["comment"] = df["comment"].apply(lambda x: clean_text(x))  # clean text
+    df = df[df["comment"] != ""].reset_index(drop=True)  # remove empty reviews
+    return df
+
+
+def clean_text(text):
+    text = uni.normalize("NFKD", text)  # normalise characters
+    text = emoji.replace_emoji(text, "")  # remove emoji
+    text = re.sub(r"(\w)\1{2,}", r"\1", text)  # repeated chars
+    text = re.sub(r"[ ]+", " ", text).strip()  # remove extra spaces
+    return text
+
+
+# LLM
+OpenAIModel = "gpt-3.5-turbo"
+llm = ChatOpenAI(model=OpenAIModel, temperature=0.1)
+
+# Embeddings
+embeddings = HuggingFaceEmbeddings(model_name="Blaxzter/LaBSE-sentence-embeddings")
+
+cache_URL = ""
+db = None
+qa = None
+
+
+def generate(URL, query):
+    global cache_URL, db, qa
+    if URL != cache_URL:
+        # Get reviews
+        try:
+            reviews = scrape(URL)
+            # Clean reviews
+            cleaned_reviews = clean(reviews)
+            # Load data
+            loader = DataFrameLoader(cleaned_reviews, page_content_column="comment")
+            documents = loader.load()
+        except Exception as e:
+            return "Error getting reviews: " + str(e)
+
+        # Split text
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1000, chunk_overlap=50
+        )
+        docs = text_splitter.split_documents(documents)
+        cache_URL = URL
+        # Vector store
+        db = FAISS.from_documents(docs, embeddings)
+        # Chain to answer questions
+        qa = RetrievalQA.from_chain_type(llm=llm, retriever=db.as_retriever())
+    return qa.run(query)
+
+
+# Gradio
+product_box = gr.Textbox(
+    label="URL Produk", placeholder="URL produk dari Shopee Indonesia"
+)
+query_box = gr.Textbox(
+    lines=2,
+    label="Kueri",
+    placeholder="Contoh: Apa yang orang katakan tentang kualitas produknya?, Bagaimana pendapat orang yang kurang puas dengan produknya?",
+)
+
+gr.Interface(
+    fn=generate,
+    inputs=[product_box, query_box],
+    outputs=gr.Textbox(label="Jawaban"),
+    title="RingkasUlas",
+    description="Bot percakapan yang bisa meringkas ulasan-ulasan produk di Shopee Indonesia (https://shopee.co.id/). Harap bersabar, bot ini dapat memakan waktu agak lama saat mengambil ulasan dari Shopee dan menyiapkan jawabannya.",
+    allow_flagging="never",
+    examples=[
+        [
+            "https://shopee.co.id/Bantal-Selimut-Balmut-Mini-Karakter-kain-CVC-i.2392232.8965506?xptdk=324a77c0-7860-4059-b00d-5d3b340f8dfe",
+            "Apa yang orang katakan tentang kualitas produknya?",
+        ],
+        [
+            "https://shopee.co.id/Bantal-Selimut-Balmut-Mini-Karakter-kain-CVC-i.2392232.8965506?xptdk=324a77c0-7860-4059-b00d-5d3b340f8dfe",
+            "Bagaimana pendapat orang yang kurang puas dengan produknya?",
+        ],
+    ],
+).launch()

tokopedia.py

@@ -0,0 +1,126 @@
+import requests
+
+ENDPOINT = "https://gql.tokopedia.com/graphql/productReviewList"
+
+def request_product_id(shop_domain, product_key):
+    endpoint = "https://gql.tokopedia.com/graphql/PDPGetLayoutQuery"
+    payload = {
+        "operationName": "PDPGetLayoutQuery",
+        "variables": {
+            "shopDomain": f"{shop_domain}",
+            "productKey": f"{product_key}",
+            "apiVersion": 1,
+        },
+        "query": """fragment ProductVariant on pdpDataProductVariant {
+                        errorCode
+                        parentID
+                        defaultChild
+                        children {
+                            productID
+                        }
+                        __typename
+                        }
+
+                        query PDPGetLayoutQuery($shopDomain: String, $productKey: String, $layoutID: String, $apiVersion: Float, $userLocation: pdpUserLocation, $extParam: String, $tokonow: pdpTokoNow, $deviceID: String) {
+                        pdpGetLayout(shopDomain: $shopDomain, productKey: $productKey, layoutID: $layoutID, apiVersion: $apiVersion, userLocation: $userLocation, extParam: $extParam, tokonow: $tokonow, deviceID: $deviceID) {
+                            requestID
+                            name
+                            pdpSession
+                            basicInfo {
+                              id: productID
+                            }
+                            components {
+                            name
+                            type
+                            position
+                            data {
+                                ...ProductVariant
+                                __typename
+                            }
+                            __typename
+                            }
+                            __typename
+                        }
+                        }
+                        """,
+    }
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
+        "Referer": "https://www.tokopedia.com",
+        "X-TKPD-AKAMAI": "pdpGetLayout",
+    }
+
+    return requests.request(method="POST", url=endpoint, json=payload, headers=headers)
+
+
+def request_product_review(product_id, page=1, limit=20):
+    payload = {
+        "operationName": "productReviewList",
+        "variables": {
+            "productID": f"{product_id}",
+            "page": page,
+            "limit": limit,
+            "sortBy": "",
+            "filterBy": "",
+        },
+        "query": """query productReviewList($productID: String!, $page: Int!, $limit: Int!, $sortBy: String, $filterBy: String) {
+  productrevGetProductReviewList(productID: $productID, page: $page, limit: $limit, sortBy: $sortBy, filterBy: $filterBy) {
+    productID
+    list {
+      id: feedbackID
+      variantName
+      message
+      productRating
+      reviewCreateTime
+      reviewCreateTimestamp
+      isReportable
+      isAnonymous
+      reviewResponse {
+        message
+        createTime
+        __typename
+      }
+      user {
+        userID
+        fullName
+        image
+        url
+        __typename
+      }
+      likeDislike {
+        totalLike
+        likeStatus
+        __typename
+      }
+      stats {
+        key
+        formatted
+        count
+        __typename
+      }
+      badRatingReasonFmt
+      __typename
+    }
+    shop {
+      shopID
+      name
+      url
+      image
+      __typename
+    }
+    hasNext
+    totalReviews
+    __typename
+  }
+}
+                        """,
+    }
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
+        "Referer": "https://www.tokopedia.com",
+        "X-TKPD-AKAMAI": "productReviewList",
+    }
+
+    return requests.request(method="POST", url=ENDPOINT, json=payload, headers=headers)