Skip to content

Commit cab09fd

Browse files
stark4n6stark4n6
authored
Merge branch 'abrignoni:main' into main
2 parents 53d8ca1 + 6da6cfd commit cab09fd

1 file changed

Lines changed: 218 additions & 13 deletions

File tree

‎scripts/artifacts/googleChat.py‎

Lines changed: 218 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,203 @@
66
from scripts.artifact_report import ArtifactHtmlReport
77
from scripts.ilapfuncs import logfunc, tsv, timeline, is_platform_windows, open_sqlite_db_readonly, utf8_in_extended_ascii, media_to_html
88

9-
9+
class Tuppsub(tuple):
10+
pass
11+
12+
class ProtectedTuple(tuple):
13+
pass
14+
15+
16+
class ProtectedList(list):
17+
pass
18+
19+
20+
class ProtectedDict(dict):
21+
pass
22+
23+
24+
class ProtectedSet(set):
25+
pass
26+
def aa_flatten_dict_tu(
27+
v,
28+
listitem,
29+
forbidden=(list, tuple, set, frozenset),
30+
allowed=(
31+
str,
32+
int,
33+
float,
34+
complex,
35+
bool,
36+
bytes,
37+
type(None),
38+
ProtectedTuple,
39+
ProtectedList,
40+
ProtectedDict,
41+
ProtectedSet,
42+
Tuppsub,
43+
),
44+
):
45+
if isinstance(v, dict) or (
46+
hasattr(v, "items") and hasattr(v, "keys")
47+
): # we check right away if it is a dict or something similar (with keys/items). If we miss something, we will
48+
# only get the keys back.
49+
for k, v2 in v.items():
50+
newtu = listitem + (k,) # we accumulate all keys in a tuple
51+
52+
# and check if there are more dicts (nested) in this dict
53+
yield from aa_flatten_dict_tu(
54+
v2, listitem=newtu, forbidden=forbidden, allowed=allowed
55+
)
56+
elif isinstance(
57+
v, forbidden
58+
): # if we have an iterable without keys (list, tuple, set, frozenset) we have to enumerate them to be able to
59+
# access the original dict values later: di['blabla'][0] instead of di['blabla']
60+
61+
for indi, v2 in enumerate(v):
62+
63+
if isinstance(v2, allowed):
64+
yield v2, listitem
65+
# if the value is not in our allowed data types, we have to check if it is an iterable
66+
else:
67+
yield from aa_flatten_dict_tu(
68+
v2,
69+
listitem=(listitem + (indi,)),
70+
forbidden=forbidden,
71+
allowed=allowed,
72+
)
73+
elif isinstance(v, allowed):
74+
# if the datatype is allowed, we yield it
75+
yield Tuppsub((v, listitem))
76+
77+
# Brute force to check if we have an iterable. We have to get all iterables!
78+
else:
79+
try:
80+
for indi2, v2 in enumerate(v):
81+
82+
try:
83+
if isinstance(v2, allowed):
84+
yield v2, listitem
85+
86+
else:
87+
yield aa_flatten_dict_tu(
88+
v2,
89+
listitem=(listitem + (indi2,)),
90+
forbidden=forbidden,
91+
allowed=allowed,
92+
)
93+
except Exception:
94+
# if there is an exception, it is probably not an iterable, so we yield it
95+
yield v2, listitem
96+
except Exception:
97+
# if there is an exception, it is probably not an iterable, so we yield it
98+
yield v, listitem
99+
100+
101+
def fla_tu(
102+
item,
103+
walkthrough=(), # accumulate nested keys
104+
forbidden=(list, tuple, set, frozenset), # forbidden to yield, need to be flattened
105+
allowed=( # Data types we don't want to touch!
106+
str,
107+
int,
108+
float,
109+
complex,
110+
bool,
111+
bytes,
112+
type(None),
113+
ProtectedTuple, #
114+
ProtectedList,
115+
ProtectedDict,
116+
ProtectedSet,
117+
Tuppsub # This is the secret - Inherit from tuple and exclude it from being flattened -
118+
# ProtectedTuple does the same thing
119+
),
120+
dict_variation=(
121+
# we don't check with isinstance(), rather with type(), that way we don't have to import collections.
122+
"collections.defaultdict",
123+
"collections.UserDict",
124+
"collections.OrderedDict",
125+
),
126+
):
127+
if isinstance(item, allowed): # allowed items, so let's yield them
128+
yield item, walkthrough
129+
elif isinstance(item, forbidden):
130+
for ini, xaa in enumerate(item):
131+
try:
132+
yield from fla_tu(
133+
xaa,
134+
walkthrough=(walkthrough + (ini,)),
135+
forbidden=forbidden,
136+
allowed=allowed,
137+
dict_variation=dict_variation,
138+
) # if we have an iterable, we check recursively for other iterables
139+
140+
except Exception:
141+
142+
yield xaa, Tuppsub(
143+
(walkthrough + Tuppsub((ini,)))
144+
) # we just yield the value (value, (key1,key2,...)) because it is probably not an iterable
145+
elif isinstance(
146+
item, dict
147+
): # we need to pass dicts to aa_flatten_dict_tu(), they need a special treatment, if not, we only get the keys from the dict back
148+
149+
yield from aa_flatten_dict_tu(
150+
item, listitem=walkthrough, forbidden=forbidden, allowed=allowed
151+
)
152+
# let's try to catch all different dict variations by using ( hasattr(item, "items") and hasattr(item, "keys").
153+
# If we dont pass it to aa_flatten_dict_tu(), we only get the keys back.
154+
#
155+
# -> (hasattr(item, "items") and hasattr(item, "keys") -> Maybe better here: elif isinstance( item, dict ):
156+
elif (str(type(item)) in dict_variation) or (
157+
hasattr(item, "items") and hasattr(item, "keys")
158+
):
159+
yield from aa_flatten_dict_tu(
160+
dict(item), listitem=walkthrough, forbidden=forbidden, allowed=allowed
161+
)
162+
163+
# isinstance(item, pd.DataFrame) maybe better?
164+
elif "DataFrame" in str(type(item)):
165+
166+
yield from aa_flatten_dict_tu(
167+
item.copy().to_dict(),
168+
# pandas needs to be converted to dict first, if not, we only get the columns back. Copying might not be necessary
169+
listitem=walkthrough,
170+
forbidden=forbidden,
171+
allowed=allowed,
172+
)
173+
174+
# # many iterables are hard to identify using isinstance() / type(), so we have to use brute force to check if it is
175+
# an iterable. If one iterable escapes, we are screwed!
176+
else:
177+
try:
178+
for ini2, xaa in enumerate(item):
179+
try:
180+
if isinstance(xaa, allowed): # yield only for allowed data types
181+
182+
yield xaa, Tuppsub(
183+
(walkthrough + (ini2,))
184+
) # yields (value, (key1,key2,...)) -> always same format -> first value, then all keys in another tuple
185+
else: # if it is not in the allowed data types, we check recursively for other iterables
186+
yield from fla_tu(
187+
xaa,
188+
walkthrough=Tuppsub(
189+
(walkthrough + Tuppsub(ini2, ))
190+
), # yields (value, (key1,key2,...))
191+
forbidden=forbidden,
192+
allowed=allowed,
193+
dict_variation=dict_variation,
194+
)
195+
except Exception:
196+
197+
yield xaa, Tuppsub(
198+
(walkthrough + (ini2,))
199+
) # in case of an exception, we yield (value, (key1,key2,...))
200+
except Exception:
201+
202+
yield item, Tuppsub(
203+
(walkthrough + Tuppsub(item, ))
204+
) # in case of an exception, we yield (value, (key1,key2,...))
205+
10206
def get_googleChat(files_found, report_folder, seeker, wrap_text, timezone_offset):
11207
for file_found in files_found:
12208
file_found = str(file_found)
@@ -74,19 +270,28 @@ def get_googleChat(files_found, report_folder, seeker, wrap_text, timezone_offse
74270
mediafilename = ''
75271
else:
76272
protostuff, types = blackboxprotobuf.decode_message(protobufmedia)
77-
checkkeyten = (protostuff['1'].get('10'))
78-
if checkkeyten is not None:
79-
mediafilename = (protostuff['1']['10']['3'])
80-
mediafilename = (mediafilename.decode())
81-
attachment = seeker.search('*/'+guid+'/tmp/'+mediafilename, return_on_first_hit=True)
82-
83-
if len(attachment) < 1:
84-
thumb = ''
85-
else:
86-
thumb = media_to_html(attachment[0], (attachment[0],), report_folder)
273+
aggregator = []
274+
if isinstance(protostuff['1'], list):
275+
nested_whatever=list(fla_tu(protostuff['1']))
276+
for x in nested_whatever:
277+
if isinstance(x[0], bytes):
278+
aggregator.append(x[0].decode())
279+
mediafilename = 'Group List'
280+
thumb = aggregator
87281
else:
88-
mediafilename = ''
89-
media = ''
282+
checkkeyten = (protostuff['1'].get('10'))
283+
if checkkeyten is not None:
284+
mediafilename = (protostuff['1']['10']['3'])
285+
mediafilename = (mediafilename.decode())
286+
attachment = seeker.search('*/'+guid+'/tmp/'+mediafilename, return_on_first_hit=True)
287+
288+
if len(attachment) < 1:
289+
thumb = ''
290+
else:
291+
thumb = media_to_html(attachment[0], (attachment[0],), report_folder)
292+
else:
293+
mediafilename = ''
294+
media = ''
90295

91296
data_list.append((row[0],row[1],row[2],row[3],row[4],mediafilename,thumb,reaction,reactionuser))
92297
mediafilename = thumb = reaction = reactionuser = ''

0 commit comments

Comments
 (0)