Arts-of-coding
commited on
Commit
•
901b6e3
1
Parent(s):
ba28f57
Update main.py
Browse files
main.py
CHANGED
@@ -24,23 +24,40 @@ config_fig = {
|
|
24 |
}
|
25 |
}
|
26 |
|
27 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
28 |
|
29 |
# Add the read-in data from the yaml file
|
30 |
-
def read_config(filename):
|
31 |
-
with open(filename, 'r') as yaml_file:
|
32 |
-
config = yaml.safe_load(yaml_file)
|
33 |
-
|
34 |
-
|
35 |
-
config = read_config(config_path)
|
36 |
-
path_parquet = config.get("path_parquet")
|
37 |
-
conditions = config.get("conditions")
|
38 |
-
col_features = config.get("col_features")
|
39 |
-
col_counts = config.get("col_counts")
|
40 |
-
col_mt = config.get("col_mt")
|
41 |
|
42 |
# Import the data from one .parquet file
|
43 |
-
df = pl.read_parquet(path_parquet)
|
44 |
#df = df.rename({"__index_level_0__": "Unnamed: 0"})
|
45 |
|
46 |
# Setup the app
|
|
|
24 |
}
|
25 |
}
|
26 |
|
27 |
+
from adlfs import AzureBlobFileSystem
|
28 |
+
mountpount=os.environ['AZURE_MOUNT_POINT'],
|
29 |
+
accountkey=os.environ['AZURE_STORAGE_ACCESS_KEY'],
|
30 |
+
accountname=os.environ['AZURE_STORAGE_ACCOUNT'],
|
31 |
+
|
32 |
+
storage_options={'account_name': ACCOUNT_NAME, 'anon': False}
|
33 |
+
df = pl.read_parquet(path="liu/filename.parquet",storage_options=storage_options)
|
34 |
+
#abfs = AzureBlobFileSystem(account_name=accountname,account_key=accountkey)
|
35 |
+
|
36 |
+
#pq.write_table(polars_dataframe.to_arrow(), 'liu/file_name.parquet', filesystem=abfs)
|
37 |
+
#storage_options = {
|
38 |
+
# "aws_access_key_id": ,
|
39 |
+
# "aws_secret_access_key": os.environ['AZURE_STORAGE_ACCESS_KEY'],
|
40 |
+
#}
|
41 |
+
#df = pl.scan_parquet(source, storage_options=storage_options)
|
42 |
+
|
43 |
+
|
44 |
+
#config_path = "./azure/data/config.yaml"
|
45 |
|
46 |
# Add the read-in data from the yaml file
|
47 |
+
#def read_config(filename):
|
48 |
+
# with open(filename, 'r') as yaml_file:
|
49 |
+
# config = yaml.safe_load(yaml_file)
|
50 |
+
# return config
|
51 |
+
|
52 |
+
#config = read_config(config_path)
|
53 |
+
#path_parquet = config.get("path_parquet")
|
54 |
+
#conditions = config.get("conditions")
|
55 |
+
#col_features = config.get("col_features")
|
56 |
+
#col_counts = config.get("col_counts")
|
57 |
+
#col_mt = config.get("col_mt")
|
58 |
|
59 |
# Import the data from one .parquet file
|
60 |
+
#df = pl.read_parquet(path_parquet)
|
61 |
#df = df.rename({"__index_level_0__": "Unnamed: 0"})
|
62 |
|
63 |
# Setup the app
|