Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import requests
|
2 |
+
import json
|
3 |
+
import time
|
4 |
+
import csv
|
5 |
+
from datetime import datetime, timedelta
|
6 |
+
from datasets import Dataset, load_dataset
|
7 |
+
import pandas as pd
|
8 |
+
from huggingface_hub import HfApi
|
9 |
+
|
10 |
+
def get_data(symbol, interval, start_time):
|
11 |
+
url = f"https://contract.mexc.com/api/v1/contract/kline/{symbol}?interval={interval}"
|
12 |
+
if start_time:
|
13 |
+
url += f"&start_time={start_time}"
|
14 |
+
|
15 |
+
try:
|
16 |
+
response = requests.get(url)
|
17 |
+
response.raise_for_status()
|
18 |
+
return response.json()
|
19 |
+
except requests.exceptions.RequestException as e:
|
20 |
+
print(f"Error during request: {e}")
|
21 |
+
return None
|
22 |
+
|
23 |
+
def process_data(data):
|
24 |
+
if not data or not data.get('data'):
|
25 |
+
return []
|
26 |
+
|
27 |
+
processed_data = []
|
28 |
+
for item in data['data']:
|
29 |
+
timestamp, open_price, high_price, low_price, close_price, volume, amount = item
|
30 |
+
processed_data.append({
|
31 |
+
"time": int(timestamp),
|
32 |
+
"open": float(open_price),
|
33 |
+
"high": float(high_price),
|
34 |
+
"low": float(low_price),
|
35 |
+
"close": float(close_price),
|
36 |
+
"volume": float(volume),
|
37 |
+
"amount": float(amount)
|
38 |
+
})
|
39 |
+
return processed_data
|
40 |
+
|
41 |
+
def create_huggingface_dataset(data, dataset_id):
|
42 |
+
df = pd.DataFrame(data)
|
43 |
+
dataset = Dataset.from_pandas(df)
|
44 |
+
dataset.push_to_hub(dataset_id)
|
45 |
+
return dataset
|
46 |
+
|
47 |
+
|
48 |
+
def main():
|
49 |
+
symbol = "BTC_USDT"
|
50 |
+
interval = "Min1"
|
51 |
+
dataset_id = "ckcl/mexc_new" # Replace with your Hugging Face Dataset ID
|
52 |
+
header = ["time", "open", "high", "low", "close", "volume", "amount"]
|
53 |
+
last_time = None
|
54 |
+
api = HfApi()
|
55 |
+
|
56 |
+
|
57 |
+
while True:
|
58 |
+
data = get_data(symbol, interval, last_time)
|
59 |
+
if data:
|
60 |
+
processed_data = process_data(data)
|
61 |
+
|
62 |
+
if processed_data:
|
63 |
+
try:
|
64 |
+
dataset = load_dataset(dataset_id) # Load from the Hub if it exists
|
65 |
+
df = pd.concat([pd.DataFrame(dataset['train']), pd.DataFrame(processed_data)], ignore_index=True)
|
66 |
+
dataset = Dataset.from_pandas(df) # append new data
|
67 |
+
except Exception as e:
|
68 |
+
print(f"Error loading dataset: {e} - create from new")
|
69 |
+
dataset = create_huggingface_dataset(processed_data, dataset_id) # create new dataset if it doesnt exist
|
70 |
+
dataset.push_to_hub(dataset_id, private = True) # Overwrite the dataset on the hub
|
71 |
+
last_time = processed_data[-1]['time']
|
72 |
+
print(f"Data saved to Huggingface Hub: {dataset_id}. last time {last_time}")
|
73 |
+
else:
|
74 |
+
print("No new data")
|
75 |
+
else:
|
76 |
+
print("Could not retrieve data")
|
77 |
+
time.sleep(60) # Check MEXC API for rate limits
|
78 |
+
if __name__ == "__main__":
|
79 |
+
main()
|