Arts-of-coding commited on
Commit
901b6e3
1 Parent(s): ba28f57

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +30 -13
main.py CHANGED
@@ -24,23 +24,40 @@ config_fig = {
24
  }
25
  }
26
 
27
- config_path = "./azure/data/config.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
 
29
  # Add the read-in data from the yaml file
30
- def read_config(filename):
31
- with open(filename, 'r') as yaml_file:
32
- config = yaml.safe_load(yaml_file)
33
- return config
34
-
35
- config = read_config(config_path)
36
- path_parquet = config.get("path_parquet")
37
- conditions = config.get("conditions")
38
- col_features = config.get("col_features")
39
- col_counts = config.get("col_counts")
40
- col_mt = config.get("col_mt")
41
 
42
  # Import the data from one .parquet file
43
- df = pl.read_parquet(path_parquet)
44
  #df = df.rename({"__index_level_0__": "Unnamed: 0"})
45
 
46
  # Setup the app
 
24
  }
25
  }
26
 
27
+ from adlfs import AzureBlobFileSystem
28
+ mountpount=os.environ['AZURE_MOUNT_POINT'],
29
+ accountkey=os.environ['AZURE_STORAGE_ACCESS_KEY'],
30
+ accountname=os.environ['AZURE_STORAGE_ACCOUNT'],
31
+
32
+ storage_options={'account_name': ACCOUNT_NAME, 'anon': False}
33
+ df = pl.read_parquet(path="liu/filename.parquet",storage_options=storage_options)
34
+ #abfs = AzureBlobFileSystem(account_name=accountname,account_key=accountkey)
35
+
36
+ #pq.write_table(polars_dataframe.to_arrow(), 'liu/file_name.parquet', filesystem=abfs)
37
+ #storage_options = {
38
+ # "aws_access_key_id": ,
39
+ # "aws_secret_access_key": os.environ['AZURE_STORAGE_ACCESS_KEY'],
40
+ #}
41
+ #df = pl.scan_parquet(source, storage_options=storage_options)
42
+
43
+
44
+ #config_path = "./azure/data/config.yaml"
45
 
46
  # Add the read-in data from the yaml file
47
+ #def read_config(filename):
48
+ # with open(filename, 'r') as yaml_file:
49
+ # config = yaml.safe_load(yaml_file)
50
+ # return config
51
+
52
+ #config = read_config(config_path)
53
+ #path_parquet = config.get("path_parquet")
54
+ #conditions = config.get("conditions")
55
+ #col_features = config.get("col_features")
56
+ #col_counts = config.get("col_counts")
57
+ #col_mt = config.get("col_mt")
58
 
59
  # Import the data from one .parquet file
60
+ #df = pl.read_parquet(path_parquet)
61
  #df = df.rename({"__index_level_0__": "Unnamed: 0"})
62
 
63
  # Setup the app