ckcl commited on
Commit
927b6f9
·
verified ·
1 Parent(s): 1983296

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +79 -0
app.py ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+ import json
3
+ import time
4
+ import csv
5
+ from datetime import datetime, timedelta
6
+ from datasets import Dataset, load_dataset
7
+ import pandas as pd
8
+ from huggingface_hub import HfApi
9
+
10
+ def get_data(symbol, interval, start_time):
11
+ url = f"https://contract.mexc.com/api/v1/contract/kline/{symbol}?interval={interval}"
12
+ if start_time:
13
+ url += f"&start_time={start_time}"
14
+
15
+ try:
16
+ response = requests.get(url)
17
+ response.raise_for_status()
18
+ return response.json()
19
+ except requests.exceptions.RequestException as e:
20
+ print(f"Error during request: {e}")
21
+ return None
22
+
23
+ def process_data(data):
24
+ if not data or not data.get('data'):
25
+ return []
26
+
27
+ processed_data = []
28
+ for item in data['data']:
29
+ timestamp, open_price, high_price, low_price, close_price, volume, amount = item
30
+ processed_data.append({
31
+ "time": int(timestamp),
32
+ "open": float(open_price),
33
+ "high": float(high_price),
34
+ "low": float(low_price),
35
+ "close": float(close_price),
36
+ "volume": float(volume),
37
+ "amount": float(amount)
38
+ })
39
+ return processed_data
40
+
41
+ def create_huggingface_dataset(data, dataset_id):
42
+ df = pd.DataFrame(data)
43
+ dataset = Dataset.from_pandas(df)
44
+ dataset.push_to_hub(dataset_id)
45
+ return dataset
46
+
47
+
48
+ def main():
49
+ symbol = "BTC_USDT"
50
+ interval = "Min1"
51
+ dataset_id = "ckcl/mexc_new" # Replace with your Hugging Face Dataset ID
52
+ header = ["time", "open", "high", "low", "close", "volume", "amount"]
53
+ last_time = None
54
+ api = HfApi()
55
+
56
+
57
+ while True:
58
+ data = get_data(symbol, interval, last_time)
59
+ if data:
60
+ processed_data = process_data(data)
61
+
62
+ if processed_data:
63
+ try:
64
+ dataset = load_dataset(dataset_id) # Load from the Hub if it exists
65
+ df = pd.concat([pd.DataFrame(dataset['train']), pd.DataFrame(processed_data)], ignore_index=True)
66
+ dataset = Dataset.from_pandas(df) # append new data
67
+ except Exception as e:
68
+ print(f"Error loading dataset: {e} - create from new")
69
+ dataset = create_huggingface_dataset(processed_data, dataset_id) # create new dataset if it doesnt exist
70
+ dataset.push_to_hub(dataset_id, private = True) # Overwrite the dataset on the hub
71
+ last_time = processed_data[-1]['time']
72
+ print(f"Data saved to Huggingface Hub: {dataset_id}. last time {last_time}")
73
+ else:
74
+ print("No new data")
75
+ else:
76
+ print("Could not retrieve data")
77
+ time.sleep(60) # Check MEXC API for rate limits
78
+ if __name__ == "__main__":
79
+ main()