Upload dict_utils.py with huggingface_hub
Browse files- dict_utils.py +465 -106
dict_utils.py
CHANGED
@@ -1,128 +1,487 @@
|
|
1 |
-
import
|
2 |
-
from typing import
|
3 |
-
|
4 |
-
|
5 |
-
|
6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
7 |
|
8 |
|
9 |
def is_subpath(subpath, fullpath):
|
10 |
-
# Normalize the paths to handle different formats and separators
|
11 |
-
subpath = os.path.normpath(subpath)
|
12 |
-
fullpath = os.path.normpath(fullpath)
|
13 |
-
|
14 |
# Split the paths into individual components
|
15 |
-
subpath_components = subpath
|
16 |
-
fullpath_components = fullpath
|
17 |
|
18 |
# Check if the full path starts with the subpath
|
19 |
return fullpath_components[: len(subpath_components)] == subpath_components
|
20 |
|
21 |
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
#
|
55 |
-
#
|
56 |
-
if
|
57 |
-
|
|
|
|
|
|
|
|
|
58 |
else:
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
65 |
else:
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
|
72 |
-
|
73 |
-
|
74 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
75 |
raise ValueError(
|
76 |
-
f
|
77 |
)
|
78 |
-
for path in paths:
|
79 |
-
dpath_set_one(dic, path, value)
|
80 |
-
|
81 |
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
-
|
|
|
|
|
|
|
|
|
86 |
raise ValueError(
|
87 |
-
f"
|
88 |
)
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
94 |
)
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
109 |
|
110 |
-
|
|
|
111 |
|
112 |
if not_exist_ok:
|
113 |
-
return
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
119 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
120 |
|
121 |
-
|
122 |
-
|
123 |
-
if set_multiple:
|
124 |
-
dpath_set_multiple(dic, query, value, not_exist_ok=not_exist_ok)
|
125 |
-
else:
|
126 |
-
dpath_set(dic, query, value, not_exist_ok=not_exist_ok)
|
127 |
-
else:
|
128 |
-
dic[query] = value
|
|
|
1 |
+
import re
|
2 |
+
from typing import Any, List, Tuple
|
3 |
+
|
4 |
+
indx = re.compile(r"^(\d+)$")
|
5 |
+
name = re.compile(r"^[\w. -]+$")
|
6 |
+
|
7 |
+
# formal definition of qpath syntax by which a query is specified:
|
8 |
+
# qpath -> A (/A)*
|
9 |
+
# A -> name | * | non-neg-int
|
10 |
+
# name -> name.matches()
|
11 |
+
# * matches ALL members (each and every) of a list or a dictionary element in input dictionary,
|
12 |
+
#
|
13 |
+
# a path p in dictionary dic is said to match query qpath if it satisfies the following recursively
|
14 |
+
# defined condition:
|
15 |
+
# (1) the prefix of length 0 of p (i.e., pref = "") matches the whole of dic. Also denoted here: pref leads to dic.
|
16 |
+
# (2) Denoting by el the element in dic lead to by prefix pref of qpath (el must be a list or dictionary),
|
17 |
+
# and by A (as the definition above) the component, DIFFERENT from *, in qpath, that follows pref, the element
|
18 |
+
# lead to by pref/A is el[A]. If el[A] is missing from dic, then no path in dic matches prefix pref/A of qpath,
|
19 |
+
# and hence no path in dic matches query qpath. (E.g., when el is a list, A must match indx, and its
|
20 |
+
# int value should be smaller than len(el) in order for the path in dic leading to element el[A] to match pref/A)
|
21 |
+
# (3) Denoting as in (2), now with A == * : when el is a list, each and every element in the set:
|
22 |
+
# {el[0], el[1], .. , el[len(el)-1]} is said to be lead to by a path matching pref/*
|
23 |
+
# and when el is a dict, each and every element in the set {el[k] for k being a key in el} is said to be lead
|
24 |
+
# to by a path matching pref/*
|
25 |
+
#
|
26 |
+
# An element el lead to by path p that matches qpath as a whole is thus either a list member (when indx.match the last
|
27 |
+
# component of p, indexing into el) or a dictionary item (the key of which equals the last component of p). The value
|
28 |
+
# of el (i.e. el[last component of p]) is returned (dic_get) or popped (dic_delete) or replaced by a new value (dic_set).
|
29 |
+
#
|
30 |
+
# Thus, for a query with no *, dic contains at most one element the path to which matches the query.
|
31 |
+
# If there is such one in dic - the function (either dict_get, dict_set, or dict_delete) operates on
|
32 |
+
# that element according to its arguments, other than not_exist_ok
|
33 |
+
# If there is not any such element in dic - the function throws or does not throw an exception, depending
|
34 |
+
# on flag not_exist_ok.
|
35 |
+
# For a query with *, there could be up to as many as there are values to match the *
|
36 |
+
# (i.e., length of the list element or dictionary element the children of which match the *; and
|
37 |
+
# for more than one * in the query -- this effect multiplies)
|
38 |
+
# Each of the three functions below (dict_get, dict_set, dict_delete) applies the requested
|
39 |
+
# operation (read, set, or delete) to each and every element el in dic, the path to which matches the query in whole,
|
40 |
+
# and reads a value from, or sets a new value to, or pops the value out from dic.
|
41 |
+
#
|
42 |
+
# If no path in dic matches the query, then # if not_exist_ok=False, the function throws an exception;
|
43 |
+
# but if not_exist_ok=True, the function returns a default value (dict_get) or does nothing (dict_delete)
|
44 |
+
# or generates all the needed missing suffixes (dict_set, see details below).
|
45 |
+
#
|
46 |
+
# Each of the functions below scans the dic-tree recursively.
|
47 |
+
# It swallows all exceptions, in order to not stop prematurely, before the scan
|
48 |
+
# has exhaustively found each and every matching path.
|
49 |
+
|
50 |
+
|
51 |
+
# validate and normalizes into components
|
52 |
+
def validate_query_and_break_to_components(query: str) -> List[str]:
|
53 |
+
if not isinstance(query, str) or len(query) == 0:
|
54 |
+
raise ValueError(
|
55 |
+
f"invalid query: either not a string or an empty string: {query}"
|
56 |
+
)
|
57 |
+
query = query.replace("//", "/").strip()
|
58 |
+
if query.startswith("/"):
|
59 |
+
query = query[1:]
|
60 |
+
# ignore the leading /, all paths are treated as coming from root of dic
|
61 |
+
|
62 |
+
if query.endswith("/"):
|
63 |
+
query = query + "*"
|
64 |
+
# same meaning, and make sure the / is not lost when splitting
|
65 |
+
|
66 |
+
components = query.split("/")
|
67 |
+
components = [component.strip() for component in components]
|
68 |
+
for component in components:
|
69 |
+
if not (
|
70 |
+
bool(name.match(component))
|
71 |
+
or component == "*"
|
72 |
+
or bool(indx.match(component))
|
73 |
+
):
|
74 |
+
raise ValueError(
|
75 |
+
f"Component {component} in input query is none of: valid field-name, non-neg-int, or '*'"
|
76 |
+
)
|
77 |
+
return components
|
78 |
|
79 |
|
80 |
def is_subpath(subpath, fullpath):
|
|
|
|
|
|
|
|
|
81 |
# Split the paths into individual components
|
82 |
+
subpath_components = validate_query_and_break_to_components(subpath)
|
83 |
+
fullpath_components = validate_query_and_break_to_components(fullpath)
|
84 |
|
85 |
# Check if the full path starts with the subpath
|
86 |
return fullpath_components[: len(subpath_components)] == subpath_components
|
87 |
|
88 |
|
89 |
+
# We are on current_element, going down from it via query[index_into_query].
|
90 |
+
# query comprising at least two components is assumed. dic_delete worries about
|
91 |
+
# single component queries without invoking qpath_delete
|
92 |
+
# Returned value is a pair (boolean, element_of_input dic or None)
|
93 |
+
# the first component signals whether the second is yielded from a reach to the query end,
|
94 |
+
# or rather -- a result of a failure before query end has been reached.
|
95 |
+
# If the first component is True, the second is current_element following a successful delete
|
96 |
+
def delete_values(
|
97 |
+
current_element: Any,
|
98 |
+
query: List[str],
|
99 |
+
index_into_query: int,
|
100 |
+
remove_empty_ancestors=False,
|
101 |
+
) -> Tuple[bool, Any]:
|
102 |
+
component = query[index_into_query]
|
103 |
+
if index_into_query == -1:
|
104 |
+
if component == "*":
|
105 |
+
# delete all members of the list or dict
|
106 |
+
current_element = [] if isinstance(current_element, list) else {}
|
107 |
+
return (True, current_element)
|
108 |
+
# component is a either a dictionary key or an index into a list,
|
109 |
+
# pop the respective element from current_element
|
110 |
+
if indx.match(component):
|
111 |
+
component = int(component)
|
112 |
+
try:
|
113 |
+
current_element.pop(component)
|
114 |
+
return (True, current_element)
|
115 |
+
except:
|
116 |
+
# no continuation in dic, from current_element down, that matches the query
|
117 |
+
return (False, None)
|
118 |
+
|
119 |
+
# index_into_query < -1
|
120 |
+
if component == "*":
|
121 |
+
# current_element must be a dict or list. We need to update value for all its members
|
122 |
+
# through which passes a path that reached the query end
|
123 |
+
if isinstance(current_element, dict):
|
124 |
+
key_values = list(current_element.items())
|
125 |
+
keys, values = zip(*key_values)
|
126 |
+
elif isinstance(current_element, list):
|
127 |
+
keys = list(range(len(current_element)))
|
128 |
+
values = current_element
|
129 |
else:
|
130 |
+
return (False, None)
|
131 |
+
|
132 |
+
any_success = False
|
133 |
+
for i in range(
|
134 |
+
len(keys) - 1, -1, -1
|
135 |
+
): # going backward to allow popping from a list
|
136 |
+
try:
|
137 |
+
success, new_val = delete_values(
|
138 |
+
current_element=values[i],
|
139 |
+
query=query,
|
140 |
+
index_into_query=index_into_query + 1,
|
141 |
+
remove_empty_ancestors=remove_empty_ancestors,
|
142 |
+
)
|
143 |
+
if not success:
|
144 |
+
continue
|
145 |
+
any_success = True
|
146 |
+
if (len(new_val) == 0) and remove_empty_ancestors:
|
147 |
+
current_element.pop(keys[i])
|
148 |
+
else:
|
149 |
+
current_element[keys[i]] = new_val
|
150 |
+
|
151 |
+
except:
|
152 |
+
continue
|
153 |
+
return (any_success, current_element)
|
154 |
+
|
155 |
+
# current component is index into a list or a key into a dictionary
|
156 |
+
if indx.match(component):
|
157 |
+
component = int(component)
|
158 |
+
try:
|
159 |
+
success, new_val = delete_values(
|
160 |
+
current_element=current_element[component],
|
161 |
+
query=query,
|
162 |
+
index_into_query=index_into_query + 1,
|
163 |
+
remove_empty_ancestors=remove_empty_ancestors,
|
164 |
+
)
|
165 |
+
if not success:
|
166 |
+
return (False, None)
|
167 |
+
if (len(new_val) == 0) and remove_empty_ancestors:
|
168 |
+
current_element.pop(component)
|
169 |
else:
|
170 |
+
current_element[component] = new_val
|
171 |
+
return (True, current_element)
|
172 |
+
except:
|
173 |
+
return (False, None)
|
174 |
+
|
175 |
+
|
176 |
+
def dict_delete(
|
177 |
+
dic: dict, query: str, not_exist_ok: bool = False, remove_empty_ancestors=False
|
178 |
+
):
|
179 |
+
# We remove from dic the value from each and every element lead to by a path matching the query.
|
180 |
+
# If remove_empty_ancestors=True, and the removal of any such value leaves its containing element (list or dict)
|
181 |
+
# within dic empty -- remove that element as well, and continue recursively
|
182 |
+
qpath = validate_query_and_break_to_components(query)
|
183 |
+
if len(qpath) == 1:
|
184 |
+
if qpath[0] in dic:
|
185 |
+
dic.pop(qpath[0])
|
186 |
+
return
|
187 |
+
if not not_exist_ok:
|
188 |
raise ValueError(
|
189 |
+
f"An attempt to delete from dictionary {dic}, an element {query}, that does not exist in the dictionary"
|
190 |
)
|
|
|
|
|
|
|
191 |
|
192 |
+
try:
|
193 |
+
success, new_val = delete_values(
|
194 |
+
current_element=dic,
|
195 |
+
query=qpath,
|
196 |
+
index_into_query=(-1) * len(qpath),
|
197 |
+
remove_empty_ancestors=remove_empty_ancestors,
|
198 |
+
)
|
199 |
+
if not success and not not_exist_ok:
|
200 |
raise ValueError(
|
201 |
+
f"An attempt to delete from dictionary {dic}, an element {query}, that does not exist in the dictionary"
|
202 |
)
|
203 |
+
except Exception as e:
|
204 |
+
raise ValueError(f"query {query} matches no path in dictionary {dic}") from e
|
205 |
+
|
206 |
+
|
207 |
+
# returns all the values sitting inside dic, in all the paths that match query_path
|
208 |
+
# if query includes * then return a list of values reached by all paths that match the query
|
209 |
+
# flake8: noqa: C901
|
210 |
+
def get_values(
|
211 |
+
current_element: Any, query: List[str], index_into_query: int
|
212 |
+
) -> Tuple[bool, Any]:
|
213 |
+
# going down from current_element through query[index_into_query].
|
214 |
+
if index_into_query == 0:
|
215 |
+
return (True, current_element)
|
216 |
+
|
217 |
+
# index_into_query < 0
|
218 |
+
component = query[index_into_query]
|
219 |
+
if component == "*":
|
220 |
+
# current_element must be a list or a dictionary
|
221 |
+
if not isinstance(current_element, (list, dict)):
|
222 |
+
return (False, None) # nothing good from here down the query
|
223 |
+
to_ret = []
|
224 |
+
if isinstance(current_element, dict):
|
225 |
+
sub_elements = list(current_element.values())
|
226 |
+
else: # a list
|
227 |
+
sub_elements = current_element
|
228 |
+
for sub_element in sub_elements:
|
229 |
+
try:
|
230 |
+
success, val = get_values(sub_element, query, index_into_query + 1)
|
231 |
+
if success:
|
232 |
+
to_ret.append(val)
|
233 |
+
except:
|
234 |
+
continue
|
235 |
+
|
236 |
+
return (len(to_ret) > 0, to_ret)
|
237 |
+
# next_component is indx or name, current_element must be a list or a dict
|
238 |
+
if indx.match(component):
|
239 |
+
component = int(component)
|
240 |
+
try:
|
241 |
+
success, new_val = get_values(
|
242 |
+
current_element[component], query, index_into_query + 1
|
243 |
)
|
244 |
+
if success:
|
245 |
+
return (True, new_val)
|
246 |
+
return (False, None)
|
247 |
+
except:
|
248 |
+
return (False, None)
|
249 |
+
|
250 |
+
|
251 |
+
# going down from current_element via query[index_into_query]
|
252 |
+
# returns the updated current_element
|
253 |
+
def set_values(
|
254 |
+
current_element: Any,
|
255 |
+
value: Any,
|
256 |
+
index_into_query: int,
|
257 |
+
fixed_parameters: dict,
|
258 |
+
set_multiple: bool = False,
|
259 |
+
) -> Tuple[bool, Any]:
|
260 |
+
if index_into_query == 0:
|
261 |
+
return (True, value) # matched query all along!
|
262 |
+
|
263 |
+
# current_element should be a list or dict: a containing element
|
264 |
+
if current_element and not isinstance(current_element, (list, dict)):
|
265 |
+
current_element = None # give it a chance to become what is needed, if allowed
|
266 |
+
|
267 |
+
if not current_element and not fixed_parameters["generate_if_not_exists"]:
|
268 |
+
return (False, None)
|
269 |
+
component = fixed_parameters["query"][index_into_query]
|
270 |
+
|
271 |
+
if component == "*":
|
272 |
+
if current_element and set_multiple:
|
273 |
+
if isinstance(current_element, dict) and len(current_element) != len(value):
|
274 |
+
return (False, None)
|
275 |
+
if isinstance(current_element, list) and len(current_element) > len(value):
|
276 |
+
return (False, None)
|
277 |
+
if len(current_element) < len(value):
|
278 |
+
if not fixed_parameters["generate_if_not_exists"]:
|
279 |
+
return (False, None)
|
280 |
+
# current_element must be a list, extend current_element to the length needed
|
281 |
+
current_element.extend([None] * (len(value) - len(current_element)))
|
282 |
+
if not current_element:
|
283 |
+
current_element = [None] * (len(value) if set_multiple else 1)
|
284 |
+
# now current_element is of size suiting value
|
285 |
+
if isinstance(current_element, dict):
|
286 |
+
keys = sorted(current_element.keys())
|
287 |
+
else:
|
288 |
+
keys = list(range(len(current_element)))
|
289 |
+
|
290 |
+
any_success = False
|
291 |
+
for i in range(len(keys)):
|
292 |
+
try:
|
293 |
+
success, new_val = set_values(
|
294 |
+
current_element=current_element[keys[i]],
|
295 |
+
value=value[i] if set_multiple else value,
|
296 |
+
index_into_query=index_into_query + 1,
|
297 |
+
set_multiple=False, # now used, not allowed again,
|
298 |
+
fixed_parameters=fixed_parameters,
|
299 |
+
)
|
300 |
+
if not success:
|
301 |
+
continue
|
302 |
+
any_success = True
|
303 |
+
current_element[keys[i]] = new_val
|
304 |
+
|
305 |
+
except:
|
306 |
+
continue
|
307 |
+
return (any_success, current_element)
|
308 |
+
|
309 |
+
# component is an index into a list or a key into a dictionary
|
310 |
+
if indx.match(component):
|
311 |
+
if current_element and not isinstance(current_element, list):
|
312 |
+
if not fixed_parameters["generate_if_not_exists"]:
|
313 |
+
return (False, None)
|
314 |
+
current_element = []
|
315 |
+
component = int(component)
|
316 |
+
if (
|
317 |
+
current_element and component >= len(current_element)
|
318 |
+
) or not current_element:
|
319 |
+
if not fixed_parameters["generate_if_not_exists"]:
|
320 |
+
return (False, None)
|
321 |
+
# extend current_element to the length needed
|
322 |
+
if not current_element:
|
323 |
+
current_element = []
|
324 |
+
current_element.extend([None] * (component + 1 - len(current_element)))
|
325 |
+
next_current_element = current_element[component]
|
326 |
+
else: # component is a key into a dictionary
|
327 |
+
if current_element and not isinstance(current_element, dict):
|
328 |
+
if not fixed_parameters["generate_if_not_exists"]:
|
329 |
+
return (False, None)
|
330 |
+
current_element = {}
|
331 |
+
if not current_element:
|
332 |
+
current_element = {}
|
333 |
+
if (
|
334 |
+
component not in current_element
|
335 |
+
and not fixed_parameters["generate_if_not_exists"]
|
336 |
+
):
|
337 |
+
return (False, None)
|
338 |
+
next_current_element = (
|
339 |
+
None if component not in current_element else current_element[component]
|
340 |
+
)
|
341 |
+
try:
|
342 |
+
success, new_val = set_values(
|
343 |
+
current_element=next_current_element,
|
344 |
+
value=value,
|
345 |
+
index_into_query=index_into_query + 1,
|
346 |
+
fixed_parameters=fixed_parameters,
|
347 |
+
set_multiple=set_multiple,
|
348 |
+
)
|
349 |
+
if success:
|
350 |
+
current_element[component] = new_val
|
351 |
+
return (True, current_element)
|
352 |
+
return (False, None)
|
353 |
+
except:
|
354 |
+
return (False, None)
|
355 |
+
|
356 |
+
|
357 |
+
# the returned values are ordered by the lexicographic order of the paths leading to them
|
358 |
+
def dict_get(
|
359 |
+
dic: dict,
|
360 |
+
query: str,
|
361 |
+
use_dpath: bool = True,
|
362 |
+
not_exist_ok: bool = False,
|
363 |
+
default: Any = None,
|
364 |
+
):
|
365 |
+
if use_dpath and "/" in query:
|
366 |
+
components = validate_query_and_break_to_components(query)
|
367 |
+
try:
|
368 |
+
success, values = get_values(dic, components, -1 * len(components))
|
369 |
+
if not success:
|
370 |
+
if not_exist_ok:
|
371 |
+
return default
|
372 |
+
raise ValueError(
|
373 |
+
f'query "{query}" did not match any item in dict: {dic}'
|
374 |
+
)
|
375 |
+
if isinstance(values, list) and len(values) == 0:
|
376 |
+
if not_exist_ok:
|
377 |
+
return default
|
378 |
+
raise ValueError(
|
379 |
+
f'query "{query}" did not match any item in dict: {dic} while not_exist_ok=False'
|
380 |
+
)
|
381 |
+
|
382 |
+
return values
|
383 |
+
|
384 |
+
except Exception as e:
|
385 |
+
if not_exist_ok:
|
386 |
+
return default
|
387 |
+
raise ValueError(
|
388 |
+
f'query "{query}" did not match any item in dict: {dic} while not_exist_ok=False'
|
389 |
+
) from e
|
390 |
|
391 |
+
if query.strip() in dic:
|
392 |
+
return dic[query.strip()]
|
393 |
|
394 |
if not_exist_ok:
|
395 |
+
return default
|
396 |
+
|
397 |
+
raise ValueError(
|
398 |
+
f'query "{query}" did not match any item in dict: {dic} while not_exist_ok=False'
|
399 |
+
)
|
400 |
+
|
401 |
+
|
402 |
+
# dict_set sets a value, 'value', which by itself, can be a dict or list or scalar, into 'dic', to become the value of
|
403 |
+
# the element the path from 'dic' head to which matches 'query'. (aka - 'the element specified by the query')
|
404 |
+
# 'the element specified by the query' is thus either a key in a dictionary, or a list member specified by its index in the list.
|
405 |
+
# Unless otherwise specified (through 'not_exist_ok=True'), the processing of 'query' by dict_set does not generate
|
406 |
+
# any new elements into 'dic'. Rather - it just sets the 'value' arg to each and every element the path to which matches
|
407 |
+
# the query. That 'value' arg, again, can be complex and involved, a dictionary or a list, or scalar, or whatever.
|
408 |
+
#
|
409 |
+
# When not_exist_ok = True, the processing itself is allowed to generate new containing elements (dictionaries, lists, or elements
|
410 |
+
# therein) into dictionary 'dic', new containing elements such that, at the end of the processing, 'dic' will contain at
|
411 |
+
# least one element the path to which matches 'query', provided that no existing value in 'dic' is modified nor popped out,
|
412 |
+
# other than the values sitting on the path along 'query' in whole.
|
413 |
+
# This generation is defined as follows.
|
414 |
+
# Having generated what is needed to have in dic an element el, lead to by prefix pref of 'query', and A (as above) is the
|
415 |
+
# component that follows pref in 'query' (i.e., pref/A is a prefix of 'query', longer than pref by one component) then:
|
416 |
+
# (1) if indx.match(A), and el existed in 'dic' before dict_set was invoked, then if el is not a list, generate an empty
|
417 |
+
# list for it: []. If len(el)>A, proceed to element el[A], and continue recursively. If len(el) <= A, extend
|
418 |
+
# el with [None]*(A+1-len(el)), and continue recursively from there, with elements that surely did not exist in dic
|
419 |
+
# before dict_set was invoked. If el did not exist in 'dic' before dict_set was invoked, then a whole new list [None]*(A+1)
|
420 |
+
# is generated, and continue from there recursively.
|
421 |
+
# (2) if not indx but name.match(A), continue in analogy with (1), with el being a dictionary now.
|
422 |
+
# (3) if A is '*', and el already exists, continue into ALL el's existing sub_elements as above. if el was not existing
|
423 |
+
# in dic before dict_set was invoked, which means it is None that we ride on from (1) or (2), then we generate
|
424 |
+
# a new [None] List (only a list, not a dict, because we have no keys to offer), and continue as above
|
425 |
+
# once the end of the query is thus reached, 'value' is returned backward on the recursion, and the elements
|
426 |
+
# that were None for a while - reshape into the needed (dict or list) element.
|
427 |
+
#
|
428 |
+
# If two or more (existing in input dic, or newly generated per not_exist_ok = True) paths in dic match the query
|
429 |
+
# all the elements lead to by these paths are assigned copies of value.
|
430 |
+
#
|
431 |
+
# If set_multiple=True, 'value' must be a list, 'query' should contain at least one * , and there should be exactly
|
432 |
+
# len(values) paths that match the query, and in this case, dict_set assigns one member of 'value' to each path.
|
433 |
+
# The matching paths are sorted alphabetically toward the above assignment.
|
434 |
+
# The processing of set_multiple=True applies to the first occurrence of * in the query, and only to it, and is
|
435 |
+
# done as follows:
|
436 |
+
# Let el denote the element lead to by prefix pref of 'query' down to one component preceding the first * in 'query'
|
437 |
+
# (depending on not_exist_ok, el can be None). If el existed in dic before dict_set is invoked (el is not None) and
|
438 |
+
# is not a list nor a dict, or is a list longer than len('value') or is a dict of len different from len('value'),
|
439 |
+
# return a failure for prefix pref/*.
|
440 |
+
# If el existed, and is a list shorted than len('value'), or did not exist at all, then if not_exist_ok= False,
|
441 |
+
# return a failure for prefix pref/*. If not_exist_ok = True, then make el into a list of length
|
442 |
+
# len('value') that starts with el (if existed) and continues into zero or more None-s, as many as needed.
|
443 |
+
# Now that el (potentially wholly or partly generated just now) is a list of length len('value'), set value='value'[i]
|
444 |
+
# as the target value for the i-th path that goes through el.
|
445 |
+
# Such a breakdown of 'value' for set_multiple=True, is done only once - on the leftmost * in 'query'.
|
446 |
+
#
|
447 |
+
def dict_set(
|
448 |
+
dic: dict,
|
449 |
+
query: str,
|
450 |
+
value: Any,
|
451 |
+
use_dpath=True,
|
452 |
+
not_exist_ok=True,
|
453 |
+
set_multiple=False,
|
454 |
+
):
|
455 |
+
if set_multiple and (
|
456 |
+
not isinstance(value, list) or len(value) == 0 or "*" not in query
|
457 |
+
):
|
458 |
+
raise ValueError(
|
459 |
+
f"set_multiple == True, and yet value, {value}, is not a list or '*' is not in query '{query}'"
|
460 |
+
)
|
461 |
+
if not use_dpath or "/" not in query:
|
462 |
+
if query.strip() in dic or not_exist_ok:
|
463 |
+
dic[query] = value
|
464 |
+
return
|
465 |
+
raise ValueError(
|
466 |
+
f"not_exist_ok=False and the single component query '{query}' is not a key in dic {dic}"
|
467 |
+
)
|
468 |
|
469 |
+
# use_dpath and "/" in query
|
470 |
+
components = validate_query_and_break_to_components(query)
|
471 |
+
fixed_parameters = {
|
472 |
+
"query": components,
|
473 |
+
"generate_if_not_exists": not_exist_ok,
|
474 |
+
}
|
475 |
+
try:
|
476 |
+
success, val = set_values(
|
477 |
+
current_element=dic,
|
478 |
+
value=value,
|
479 |
+
index_into_query=(-1) * len(components),
|
480 |
+
fixed_parameters=fixed_parameters,
|
481 |
+
set_multiple=set_multiple,
|
482 |
+
)
|
483 |
+
if not success and not not_exist_ok:
|
484 |
+
raise ValueError(f"No path in dic {dic} matches query {query}.")
|
485 |
|
486 |
+
except Exception as e:
|
487 |
+
raise ValueError(f"No path in dic {dic} matches query {query}.") from e
|
|
|
|
|
|
|
|
|
|
|
|