kensvin commited on
Commit
f87f358
1 Parent(s): c150ea3

try combining into one app

Browse files
Files changed (1) hide show
  1. app.py +133 -1
app.py CHANGED
@@ -14,7 +14,7 @@ from langchain_community.document_loaders import DataFrameLoader
14
  from langchain.text_splitter import RecursiveCharacterTextSplitter
15
  from langchain_community.vectorstores import FAISS
16
  from langchain.chains import RetrievalQA
17
- from tokopedia import request_product_id, request_product_review
18
  import gradio as gr
19
 
20
  shop_id = ""
@@ -34,6 +34,138 @@ logging.basicConfig(
34
  logger = logging.getLogger(__name__)
35
 
36
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  def scrape(product_id, max_reviews=LIMIT):
38
  all_reviews = []
39
  page = 1
 
14
  from langchain.text_splitter import RecursiveCharacterTextSplitter
15
  from langchain_community.vectorstores import FAISS
16
  from langchain.chains import RetrievalQA
17
+ # from tokopedia import request_product_id, request_product_review
18
  import gradio as gr
19
 
20
  shop_id = ""
 
34
  logger = logging.getLogger(__name__)
35
 
36
 
37
+ import requests
38
+
39
+
40
+ def request_product_id(shop_domain, product_key):
41
+ endpoint = "https://gql.tokopedia.com/graphql/PDPGetLayoutQuery"
42
+ payload = {
43
+ "operationName": "PDPGetLayoutQuery",
44
+ "variables": {
45
+ "shopDomain": f"{shop_domain}",
46
+ "productKey": f"{product_key}",
47
+ "apiVersion": 1,
48
+ },
49
+ "query": """fragment ProductVariant on pdpDataProductVariant {
50
+ errorCode
51
+ parentID
52
+ defaultChild
53
+ children {
54
+ productID
55
+ }
56
+ __typename
57
+ }
58
+
59
+ query PDPGetLayoutQuery($shopDomain: String, $productKey: String, $layoutID: String, $apiVersion: Float, $userLocation: pdpUserLocation, $extParam: String, $tokonow: pdpTokoNow, $deviceID: String) {
60
+ pdpGetLayout(shopDomain: $shopDomain, productKey: $productKey, layoutID: $layoutID, apiVersion: $apiVersion, userLocation: $userLocation, extParam: $extParam, tokonow: $tokonow, deviceID: $deviceID) {
61
+ requestID
62
+ name
63
+ pdpSession
64
+ basicInfo {
65
+ id: productID
66
+ }
67
+ components {
68
+ name
69
+ type
70
+ position
71
+ data {
72
+ ...ProductVariant
73
+ __typename
74
+ }
75
+ __typename
76
+ }
77
+ __typename
78
+ }
79
+ }
80
+ """,
81
+ }
82
+
83
+ headers = {
84
+ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
85
+ "Referer": "https://www.tokopedia.com",
86
+ "X-TKPD-AKAMAI": "pdpGetLayout",
87
+ }
88
+
89
+ return requests.request(
90
+ method="POST", url=endpoint, json=payload, headers=headers, timeout=30
91
+ )
92
+
93
+
94
+ def request_product_review(product_id, page=1, limit=20):
95
+ ENDPOINT = "https://gql.tokopedia.com/graphql/productReviewList"
96
+ payload = {
97
+ "operationName": "productReviewList",
98
+ "variables": {
99
+ "productID": f"{product_id}",
100
+ "page": page,
101
+ "limit": limit,
102
+ "sortBy": "",
103
+ "filterBy": "",
104
+ },
105
+ "query": """query productReviewList($productID: String!, $page: Int!, $limit: Int!, $sortBy: String, $filterBy: String) {
106
+ productrevGetProductReviewList(productID: $productID, page: $page, limit: $limit, sortBy: $sortBy, filterBy: $filterBy) {
107
+ productID
108
+ list {
109
+ id: feedbackID
110
+ variantName
111
+ message
112
+ productRating
113
+ reviewCreateTime
114
+ reviewCreateTimestamp
115
+ isReportable
116
+ isAnonymous
117
+ reviewResponse {
118
+ message
119
+ createTime
120
+ __typename
121
+ }
122
+ user {
123
+ userID
124
+ fullName
125
+ image
126
+ url
127
+ __typename
128
+ }
129
+ likeDislike {
130
+ totalLike
131
+ likeStatus
132
+ __typename
133
+ }
134
+ stats {
135
+ key
136
+ formatted
137
+ count
138
+ __typename
139
+ }
140
+ badRatingReasonFmt
141
+ __typename
142
+ }
143
+ shop {
144
+ shopID
145
+ name
146
+ url
147
+ image
148
+ __typename
149
+ }
150
+ hasNext
151
+ totalReviews
152
+ __typename
153
+ }
154
+ }
155
+ """,
156
+ }
157
+
158
+ headers = {
159
+ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
160
+ "Referer": "https://www.tokopedia.com",
161
+ "X-TKPD-AKAMAI": "productReviewList",
162
+ }
163
+
164
+ return requests.request(
165
+ method="POST", url=ENDPOINT, json=payload, headers=headers, timeout=30
166
+ )
167
+
168
+
169
  def scrape(product_id, max_reviews=LIMIT):
170
  all_reviews = []
171
  page = 1