Skip to content

Commit 39ad790

Browse files
committed
python copy datasets to raw_data
1 parent 7d2654f commit 39ad790

13 files changed

Lines changed: 168 additions & 118 deletions

‎00_raw_data/extract_here.zip‎

-20 MB
Binary file not shown.

‎02_data_preparation/data_preparation.R‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎02_data_preparation/data_preparation.py‎

Lines changed: 0 additions & 90 deletions
This file was deleted.

‎scripts-R/data_read.Rmd‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,21 @@ output: html_document
66
---
77

88
```{r "knitr config", cache = FALSE, include=FALSE}
9-
require("knitr")
10-
knitr::opts_knit$set(root.dir = "C:/Github/Programming_Data_Science/ourdata")
11-
#lh_root
9+
#nico
10+
#require("knitr")
11+
#knitr::opts_knit$set(root.dir = "C:/Github/Programming_Data_Science/ourdata")
12+
13+
14+
#Philipp_roots
1215
#knitr::opts_knit$set(root.dir = "C:\\Users\\Philipp\\Documents\\StudDocs\\Master\\SS2019\\Programming Data Science\\data")
16+
17+
#lh_roots
18+
##surface
19+
#knitr::opts_knit$set(root.dir = "C:/Github/Programming_Data_Science/ourdata")
20+
21+
##desktop
22+
knitr::opts_knit$set(root.dir = "F:\\Laure\\Documents\\git hub Repositories\\phyk_data")
23+
1324
```
1425
```{r setup, include=FALSE}
1526
knitr::opts_chunk$set(echo = TRUE)
@@ -73,7 +84,7 @@ for (i in 1:ncol(dataframe)) {
7384
}
7485
}
7586
}
76-
# Lösche doppelte Spalten (immer die 2.)
87+
# L?sche doppelte Spalten (immer die 2.)
7788
dataframe <- subset(dataframe, select = -drop)
7889
head(dataframe)
7990
```
-509 Bytes
Binary file not shown.
-1.96 KB
Binary file not shown.
-3.46 KB
Binary file not shown.
-49 Bytes
Binary file not shown.

‎scripts-python/clean_dataframe.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,12 @@ def delete_empty_indexorcolumns_in_df(df,index_or_columns,threshold):
3737

3838

3939
def clean_df_generic(df,threshold):
40-
print(df)
40+
#print(df)
4141
index_or_columns = "columns"
4242
df = delete_empty_indexorcolumns_in_df(df,index_or_columns,threshold)
4343
index_or_columns = "index"
4444
df = delete_empty_indexorcolumns_in_df(df,index_or_columns,threshold)
45-
print(df)
45+
#print(df)
4646
return df
4747

4848

@@ -69,11 +69,11 @@ def merge_columns(df,column_name1,column_name2):
6969

7070
#order
7171
df = read_data.load_dataframe("order.pkl")
72-
df = clean_df_generic(df,1)
72+
df = clean_df_generic(df,0)
7373

7474
#clickstream
7575
df2 = read_data.load_dataframe("click.pkl")
76-
df2 = clean_df_generic(df2,1)
76+
df2 = clean_df_generic(df2,0)
7777

7878

7979

‎scripts-python/copy_open_zip.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import os.path
2+
import zipfile
3+
import os
4+
import csv
5+
import shutil
6+
from zipfile import ZipFile
7+
from absolute_data_paths import relative_to_absolute
8+
9+
def extra_zip_to(filename,destination):
10+
filename = relative_to_absolute(filename)
11+
with ZipFile(filename,"r") as zip:
12+
zfile = zip.namelist()[0]
13+
zip.extract(zfile,destination)
14+
15+
16+
def identify_and_tret(source_folder,filename):
17+
path_file = source_folder+"\\"+filename
18+
ending = filename.split(".")[1]
19+
if ending =="txt":
20+
shutil.copy(path_file,relative_to_absolute("../01_data_understanding"))
21+
elif ending =="csv":
22+
shutil.copy(path_file,relative_to_absolute("../00_raw_data"))
23+
elif ending =="zip":
24+
extra_zip_to(path_file,relative_to_absolute("../00_raw_data"))
25+
26+
27+
28+
29+
def get_files_from_folder(source_folder):
30+
folderpath = relative_to_absolute(source_folder)
31+
for path,list_folder,list_filename in os.walk(folderpath):
32+
if path == folderpath and list_folder == []:
33+
for row in list_filename:
34+
identify_and_tret(folderpath,row)
35+
36+
37+
def copy_into_raw_data_and_data_understanding():
38+
#clickstream
39+
clickstream_path = "../clickstream"
40+
get_files_from_folder(clickstream_path)
41+
#order
42+
order_path = "../orders"
43+
get_files_from_folder(order_path)
44+
45+
46+
47+

0 commit comments

Comments
 (0)