Skip to content

Commit a6a008d

Browse files
LaurenzHLaurenzH
committed
python copy datasets to raw_data
1 parent 4dfa0dc commit a6a008d

10 files changed

Lines changed: 121 additions & 169 deletions

File tree

‎00_raw_data/extract_here.zip‎

20 MB
Binary file not shown.

‎02_data_preparation/data_preparation.R‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
library(tidyverse)
3+
4+
mpg
5+
6+
ggplot(data = <DATA>)+
7+
<GEOM_FUNCTION>(mapping = aes(<MAPPINGS>))
8+

‎02_data_preparation/data_preparation.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import pandas as pd
2+
import numpy as np
3+
import os.path
4+
import csv
5+
import pickle
6+
import datetime
7+
8+
9+
def read_raw_data(filename,column_file,samplename,dataset_name):
10+
filename = "./00_raw_data/"+filename
11+
column_list = read_raw_columns(column_file)
12+
df = pd.read_csv(filename, names=column_list,low_memory=False, encoding="ansi")
13+
create_sampel_excel(df,samplename)
14+
save_dataframe(df,dataset_name)
15+
print(df)
16+
17+
18+
19+
20+
def read_raw_columns(filename):
21+
filename = "./01_data_understanding/"+filename
22+
textfull= open(filename,"r")
23+
line_list = textfull.readlines()
24+
column_list = []
25+
for line in line_list:
26+
string = line
27+
string_list = string.split(":")
28+
column_list.append(string_list[0])
29+
return column_list
30+
31+
32+
33+
34+
def create_sampel_excel(dataframe,samplename):
35+
from pandas import ExcelWriter
36+
#create a Excelsheet with the heads and the first 100 lines for data unterstanding
37+
df = dataframe.iloc[:1000]
38+
samplename = "./01_data_understanding/"+samplename
39+
xlsx_writer = ExcelWriter(samplename)
40+
df.to_excel(xlsx_writer,"Sheet1",index=False)
41+
xlsx_writer.save()
42+
43+
44+
45+
46+
def create_csv_from_dataframe(dataframe,filename):
47+
filename = "./00_raw_data/"+filename
48+
df = dataframe
49+
df.to_csv(path_or_buf=filename,index=False)
50+
51+
52+
53+
54+
def save_dataframe(dataframe,filename):
55+
filename = "./00_raw_data/data_sets/"+filename
56+
dataframe.to_pickle(filename)
57+
58+
59+
60+
61+
def load_dataframe(filename):
62+
filename = "./00_raw_data/data_sets/"+filename
63+
df = pd.DataFrame()
64+
df= pd.read_pickle(filename)
65+
return df
66+
67+
68+
#Order
69+
####
70+
##create new csv with head and data
71+
##And create sample Excelsheet with 100 lines
72+
read_raw_data("order_data.csv","order_columns.txt","order.xlsx","order.pkl")
73+
74+
75+
76+
#clickstream
77+
####
78+
##create new csv with head and data
79+
##And create sample Excelsheet with 100 lines
80+
read_raw_data("clickstream_data.csv","clickstream_columns.txt","clickstream.xlsx","click.pkl")
81+
82+
83+
84+
85+
86+
87+
88+
89+
90+

‎scripts-R/data_read.Rmd‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,10 @@ output: html_document
66
---
77

88
```{r "knitr config", cache = FALSE, include=FALSE}
9-
#nico
10-
#require("knitr")
11-
#knitr::opts_knit$set(root.dir = "C:/Github/Programming_Data_Science/ourdata")
12-
13-
14-
#Philipp_roots
9+
require("knitr")
10+
knitr::opts_knit$set(root.dir = "C:/Github/Programming_Data_Science/ourdata")
11+
#lh_root
1512
#knitr::opts_knit$set(root.dir = "C:\\Users\\Philipp\\Documents\\StudDocs\\Master\\SS2019\\Programming Data Science\\data")
16-
17-
#lh_roots
18-
##surface
19-
#knitr::opts_knit$set(root.dir = "C:/Github/Programming_Data_Science/ourdata")
20-
21-
##desktop
22-
knitr::opts_knit$set(root.dir = "F:\\Laure\\Documents\\git hub Repositories\\phyk_data")
23-
2413
```
2514
```{r setup, include=FALSE}
2615
knitr::opts_chunk$set(echo = TRUE)
@@ -84,7 +73,7 @@ for (i in 1:ncol(dataframe)) {
8473
}
8574
}
8675
}
87-
# L?sche doppelte Spalten (immer die 2.)
76+
# Lösche doppelte Spalten (immer die 2.)
8877
dataframe <- subset(dataframe, select = -drop)
8978
head(dataframe)
9079
```

‎scripts-python/__pycache__/read_data.cpython-37.pyc‎

47 Bytes
Binary file not shown.

‎scripts-python/clean_dataframe.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,12 @@ def delete_empty_indexorcolumns_in_df(df,index_or_columns,threshold):
3737

3838

3939
def clean_df_generic(df,threshold):
40-
#print(df)
40+
print(df)
4141
index_or_columns = "columns"
4242
df = delete_empty_indexorcolumns_in_df(df,index_or_columns,threshold)
4343
index_or_columns = "index"
4444
df = delete_empty_indexorcolumns_in_df(df,index_or_columns,threshold)
45-
#print(df)
45+
print(df)
4646
return df
4747

4848

@@ -68,12 +68,12 @@ def merge_columns(df,column_name1,column_name2):
6868
##generic cleaning
6969

7070
#order
71-
#df = read_data.load_dataframe("order.pkl")
72-
#df = clean_df_generic(df,0)
71+
df = read_data.load_dataframe("order.pkl")
72+
df = clean_df_generic(df,1)
7373

7474
#clickstream
75-
#df2 = read_data.load_dataframe("click.pkl")
76-
#df2 = clean_df_generic(df2,0)
75+
df2 = read_data.load_dataframe("click.pkl")
76+
df2 = clean_df_generic(df2,1)
7777

7878

7979

‎scripts-python/copy_open_zip.py‎

Lines changed: 0 additions & 49 deletions
This file was deleted.

‎scripts-python/initialize_data.py‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,15 @@
1-
from copy_open_zip import copy_into_raw_data_and_data_understanding
21
from read_data import read_raw_data
3-
from clean_dataframe import clean_df_generic
4-
5-
##get raw_datasets
6-
7-
copy_into_raw_data_and_data_understanding()
82

93
#Order
104
####
115
##create new csv with head and data
126
##And create sample Excelsheet with 100 lines
13-
df_order = read_raw_data("order_data.csv","order_columns.txt","order.xlsx","order.pkl")
14-
## clean
15-
df_order = clean_df_generic(df_order,0)
7+
read_raw_data("order_data.csv","order_columns.txt","order.xlsx","order.pkl")
168

17-
print(df_order.head(10))
189

1910

2011
#clickstream
2112
####
2213
##create new csv with head and data
2314
##And create sample Excelsheet with 100 lines
24-
df_clickstream = read_raw_data("clickstream_data.csv","clickstream_columns.txt","clickstream.xlsx","click.pkl")
25-
## clean
26-
df_clickstream = clean_df_generic(df_clickstream,0)
27-
28-
print(df_clickstream.head(10))
29-
30-
15+
read_raw_data("clickstream_data.csv","clickstream_columns.txt","clickstream.xlsx","click.pkl")

‎scripts-python/read_data.py‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,16 @@
1010
def read_raw_data(filename,column_file,samplename,dataset_name):
1111
filename = "../00_raw_data/"+filename
1212
path_to_file = relative_to_absolute(filename)
13-
columns = read_raw_columns(column_file)
13+
columns, dtypes = read_raw_columns(column_file)
1414
df = pd.read_csv(path_to_file, names=columns,low_memory=False, encoding="ansi")
15+
### deactivated
16+
# due to unwanted data altering
17+
# see for example column DoYouPurchaseForOthers before and after execution
18+
#df = recreate_dtypes(df, dtypes)
19+
### end of deactivation
20+
#create_sampel_excel(df,samplename)
1521
save_dataframe(df,dataset_name)
16-
#print(df)
22+
print(df)
1723
return df
1824

1925
def read_raw_columns(filename):
@@ -28,7 +34,7 @@ def read_raw_columns(filename):
2834
string_list = string.split(":")
2935
column_list.append(string_list[0])
3036
dtype_list.append(identify_column_type(string_list[1]))
31-
return column_list
37+
return column_list, dtype_list
3238

3339
def load_dataframe_without_ignore(filename, column_file):
3440
#Determine which columns should be ignored
@@ -93,7 +99,7 @@ def split_ordered_categories(categories):
9399

94100
def create_sampel_excel(dataframe,samplename):
95101
from pandas import ExcelWriter
96-
#create a Excelsheet with the heads and the first 1000 lines for data unterstanding
102+
#create a Excelsheet with the heads and the first 100 lines for data unterstanding
97103
df = dataframe.iloc[:1000]
98104
samplename = "../01_data_understanding/"+samplename
99105
path_to_file = relative_to_absolute(samplename)
@@ -120,6 +126,6 @@ def load_dataframe(filename):
120126

121127

122128

123-
#test if dataset is altered by type change
129+
##test if dataset is altered by type change
124130
#df = read_raw_data("order_data.csv","order_columns.txt","order.xlsx","order.pkl")
125131
#print(df[["DoYouPurchaseForOthers","SendEmail","ProductCode","New Bank Card"]])

0 commit comments

Comments
 (0)