Skip to content

Commit 9c0b62b

Browse files
alvaroofalvaroof
committed
Add folder for python training
1 parent 284cd62 commit 9c0b62b

37 files changed

Lines changed: 1251 additions & 0 deletions

‎scripts/python_training/__init__.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# -*- coding: utf-8 -*-
2+
"""Top level package for Novartis Python Training."""
3+
import os
4+
from importlib import metadata
5+
from pathlib import Path
6+
7+
__version__ = metadata.version("python_training")
8+
9+
10+
# Base path of python_training module
11+
# (to be used when accessing non .py files in Novartis Python Training/)
12+
WORKDIR = Path(os.getenv("WORKDIR", Path.cwd()))
13+
BASEPATH = Path(__file__).parent
14+
ASSET_DIR = BASEPATH / "assets"
15+
M3_PATH = BASEPATH / "m3_internal_libraries"

‎scripts/python_training/__main__.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# -*- coding: utf-8 -*-
2+
# ruff: noqa: D401
3+
"""Entry point."""
4+
import click
5+
6+
from . import __version__
7+
from .scripts import dummy
8+
9+
10+
def _main() -> None:
11+
"""Run main function for entrypoint."""
12+
13+
@click.group(chain=True)
14+
@click.version_option(__version__)
15+
def entry_point() -> None:
16+
"""Package entry point."""
17+
18+
entry_point.add_command(dummy.main)
19+
20+
entry_point()
21+
22+
23+
if __name__ == "__main__":
24+
_main()

‎scripts/python_training/m1_python_basics/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# -*- coding: utf-8 -*-
2+
"""Tasks for the module 1: Python basics."""

‎scripts/python_training/m1_python_basics/task_1.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# -*- coding: utf-8 -*-
2+
"""MODULE 1: Python Basics.
3+
4+
TASK 1: This task is to practice the basic concepts learnt in the first module, found here:
5+
<link to the pylosophy book repo's chapter of OOP and Imperative Programming>
6+
7+
Create a python class called "Vehicle". The class will have the following attributes:
8+
* Brand
9+
* Color
10+
* Max_Capacity
11+
* Speed
12+
* Power_Source
13+
* Current_Position
14+
15+
The class will also have the following functions:
16+
* Accelerate (acceleration, time)
17+
* Brake (acceleration, time)
18+
* Repaint (new_color)
19+
* Maintain_Speed (time)
20+
"""

‎scripts/python_training/m1_python_basics/task_2.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# -*- coding: utf-8 -*-
2+
"""MODULE 1: Python Basics.
3+
4+
TASK 2: This task is to practice the more advanced concepts learnt in the first module, found here:
5+
<link to the pylosophy book repo's chapter of OOP and Imperative Programming>
6+
7+
Create several python classes that inherit from Vehicle. The classes are:
8+
* CAR
9+
- Attributes:
10+
* Wheels
11+
* License_Plate
12+
* Max_Fuel
13+
* Fuel_Left
14+
* Max_Trunk_Space
15+
* Trunk_Space_Left
16+
* Consumption_Per_Km
17+
- Functions:
18+
* Register_License (new_license)
19+
* Refuel (liters)
20+
* Fill_Trunk (volume)
21+
* Empty_Trunk (volume)
22+
23+
* BOAT
24+
- Attributes:
25+
* Engines
26+
* Food_Supply_Kg
27+
* Passengers
28+
* Destination
29+
* Crew [formed by several employees: A captain, a chief engineer, a deckhand and a mate]
30+
- Functions:
31+
* Board (num_passengers)
32+
* Ressupply (kg_food)
33+
* Dock ()
34+
* Update_destination (new_dest)
35+
36+
* BIKE
37+
- Attributes:
38+
* Wheels
39+
* Owner
40+
* Size
41+
* Bike_Type
42+
* Gear_Transmission
43+
- Functions:
44+
* Change_Owner (new_owner)
45+
* Change_Transmission (new_trans)
46+
"""

‎scripts/python_training/m2_external_libraries/__init__.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# -*- coding: utf-8 -*-
2+
"""Subpackage to implement the methods for Python external libraries training."""
3+
4+
from ._preprocess import preprocess
5+
from ._utils import db_connector
6+
7+
__all__ = ["db_connector", "preprocess"]

‎scripts/python_training/m2_external_libraries/_preprocess.py‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
# -*- coding: utf-8 -*-
2+
"""Functions for testing exercises.
3+
4+
This module is designed to perform various manipulations and analyses on stock price data.
5+
The data is expected to be in a pandas DataFrame with the following columns:
6+
7+
- Date: The date of the stock data, in 'YYYY-MM-DD' format.
8+
- Open: The opening price of the stock for the day.
9+
- High: The highest price of the stock for the day.
10+
- Low: The lowest price of the stock for the day.
11+
- Close: The closing price of the stock for the day.
12+
- Adj Close: The adjusted closing price of the stock for the day (adjusted for splits and dividends).
13+
- Volume: The number of shares traded during the day.
14+
15+
DataFrame example:
16+
17+
Date Open High Low Close Adj Close Volume
18+
2020-01-01 100 110 95 105 105 10000
19+
2020-01-02 105 115 100 110 110 15000
20+
...
21+
22+
These functions are designed to prepare stock price data for analysis and modeling.
23+
They enable users to clean and transform the data, extract meaningful features, and ensure the data is
24+
in a suitable format for various analytical tasks.
25+
"""
26+
27+
from typing import Union
28+
29+
import pandas as pd
30+
31+
from ._utils import db_connector
32+
33+
34+
def preprocess(
35+
start_date: Union[str, pd.Timestamp],
36+
end_date: Union[str, pd.Timestamp],
37+
window: int = 20,
38+
lag_days: int = 5,
39+
):
40+
"""Applies a series of preprocessing steps to stock price data.
41+
42+
This function integrates several preprocessing steps: initial data preparation and normalization,
43+
technical indicator enhancement, and the addition of lag features. It's designed to prepare stock
44+
price data for further analysis or modeling.
45+
46+
:param df: The input DataFrame containing stock price data.
47+
:param start_date: The start date for filtering the data, in 'YYYY-MM-DD' format.
48+
:param end_date: The end date for filtering the data, in 'YYYY-MM-DD' format.
49+
:param window: The window size for calculating moving averages and volatility. Defaults to 20.
50+
:param lag_days: The number of days to create lag features for. Defaults to 5.
51+
52+
:returns: The processed DataFrame, ready for analysis or modeling, with normalized dates,
53+
filtered by the specified date range, enhanced with technical indicators, and augmented with lag features.
54+
"""
55+
df = db_connector().read_from_db()
56+
57+
if df is None:
58+
return None
59+
60+
return (
61+
df.pipe(_prepare_and_normalize_data, start_date=start_date, end_date=end_date)
62+
.pipe(_enhance_with_technical_indicators, window=window)
63+
.pipe(_add_lags, lag_days=lag_days)
64+
)
65+
66+
67+
def _prepare_and_normalize_data(
68+
df: pd.DataFrame, start_date: Union[str, pd.Timestamp], end_date: Union[str, pd.Timestamp]
69+
):
70+
"""Prepares and normalizes stock price data for analysis.
71+
72+
This function standardizes the date format, filters the dataset based on a specified date range,
73+
generates additional date-related features, calculates daily returns, and ensures that the dataset
74+
has no missing dates within the specified range by forward filling missing values. This comprehensive
75+
preprocessing step is crucial for subsequent financial analysis and modeling tasks.
76+
77+
:param df: DataFrame with stock price data, expecting columns 'Date' and 'Adj Close'.
78+
:param start_date: String representing the start date in 'YYYY-MM-DD' format.
79+
:param end_date: String representing the end date in 'YYYY-MM-DD' format.
80+
81+
:return: A DataFrame that has been processed to include only the specified date range,
82+
with added 'Month' and 'DayOfWeek' features, daily returns calculated, and missing dates
83+
forward-filled. The DataFrame is indexed by the 'Date' column.
84+
85+
:raises: ValueError: If the input DataFrame does not contain a 'Date' column or if the 'Date' column
86+
cannot be converted to datetime format. Also raised if the specified start_date or end_date
87+
are not in the correct format.
88+
"""
89+
if "Date" not in df.columns:
90+
raise ValueError("DataFrame must contain a 'Date' column.")
91+
try:
92+
df["Date"] = pd.to_datetime(df["Date"])
93+
except ValueError as e:
94+
raise ValueError(
95+
"Error converting 'Date' column to datetime. Ensure the dates are in a correct format."
96+
) from e
97+
98+
# Filter Date Range
99+
start_date = pd.to_datetime(start_date)
100+
end_date = pd.to_datetime(end_date)
101+
df = df[(df["Date"] >= start_date) & (df["Date"] <= end_date)]
102+
103+
# Generate Date Features
104+
df["Month"] = df["Date"].dt.month
105+
df["DayOfWeek"] = df["Date"].dt.dayofweek
106+
107+
# Calculate Daily Returns
108+
df["Daily Returns"] = df["Adj Close"].pct_change()
109+
110+
# Fill Missing Dates (forward fill for simplicity)
111+
df = df.set_index("Date").resample("D").ffill()
112+
113+
return df
114+
115+
116+
def _enhance_with_technical_indicators(df: pd.DataFrame, window: int = 20):
117+
"""Enriches a DataFrame with key technical indicators to aid in financial analysis.
118+
119+
This function calculates and appends several technical indicators to the input DataFrame,
120+
specifically a moving average and volatility measure. These indicators are widely used in
121+
stock market analysis to understand price trends and market volatility over a specified
122+
window of time. Additionally, the function ensures that the necessary prerequisites for
123+
calculation are met, such as the presence of required columns.
124+
125+
:param df: DataFrame after initial preprocessing, with 'Date' and 'Close' columns.
126+
:param window: Integer representing the window size for the moving average.
127+
128+
:return: The original DataFrame augmented with two new columns:
129+
- 'MA_{window}': The moving average of the 'Close' prices over the specified window.
130+
- 'Volatility': The rolling standard deviation of 'Daily Returns', representing price
131+
volatility over the same window.
132+
133+
:raises:
134+
- ValueError: If the input DataFrame lacks the required 'Close' or 'Daily Returns' columns,
135+
or if the 'window' parameter is not a positive integer.
136+
- RuntimeError: If an unexpected error occurs during the calculation of technical indicators.
137+
"""
138+
# Validate required columns
139+
required_columns = ["Close", "Daily Returns"]
140+
missing_columns = [col for col in required_columns if col not in df.columns]
141+
if missing_columns:
142+
raise ValueError(
143+
f"DataFrame is missing required columns for technical indicator calculations: {', '.join(missing_columns)}"
144+
)
145+
146+
# Validate window size
147+
if not isinstance(window, int) or window <= 0:
148+
raise ValueError("'window' must be a positive integer.")
149+
150+
try:
151+
# Add Moving Average
152+
df[f"MA_{window}"] = df["Close"].rolling(window=window).mean()
153+
# Calculate Volatility (rolling standard deviation of daily returns)
154+
df["Volatility"] = df["Daily Returns"].rolling(window=window).std()
155+
except Exception as e:
156+
raise RuntimeError(f"An error occurred while calculating technical indicators: {e}")
157+
158+
return df
159+
160+
161+
def _add_lags(df: pd.DataFrame, lag_days: int = 5):
162+
"""Enhances a DataFrame by adding lagged features for the 'Close' price column.
163+
164+
This function creates new columns in the DataFrame, each representing the 'Close' price
165+
shifted by a number of days specified by the 'lag_days' parameter. These lagged features
166+
are useful for time series analysis and forecasting models, as they allow the model to
167+
consider historical price movements.
168+
169+
:param df: DataFrame with stock price data.
170+
:param lag_days: Number of days to lag features by.
171+
172+
:return: A DataFrame identical to the input but with additional columns for each
173+
lagged feature. The names of these new columns follow the pattern 'Close_lag_X', where
174+
X is the number of days the 'Close' price is lagged by.
175+
176+
:raises: ValueError: If the input DataFrame does not contain a 'Close' column or if the 'lag_days'
177+
parameter is not a positive integer.
178+
"""
179+
# Validate 'Close' column presence
180+
if "Close" not in df.columns:
181+
raise ValueError("DataFrame must contain a 'Close' column for lag feature calculations.")
182+
183+
# Validate lag_days parameter
184+
if not isinstance(lag_days, int) or lag_days <= 0:
185+
raise ValueError("'lag_days' must be a positive integer.")
186+
187+
try:
188+
# Create Lag Features for 'Close' Price
189+
for i in range(1, lag_days + 1):
190+
df[f"Close_lag_{i}"] = df["Close"].shift(i)
191+
except Exception as e:
192+
# Catching a broad exception to handle unexpected errors during lag feature creation
193+
raise RuntimeError(f"An error occurred while adding lag features: {e}")
194+
195+
return df

‎scripts/python_training/m2_external_libraries/_utils.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# -*- coding: utf-8 -*-
2+
from loguru import logger
3+
4+
5+
class db_connector:
6+
def __init__(self) -> None:
7+
pass
8+
9+
def read_from_db(self):
10+
"""This function is a placeholder for database reading functionality. It's not implemented
11+
yet and is intended to be mocked during testing to return a DataFrame with stock price data.
12+
13+
:returns: None. Placeholder return value. When mocked, this should return a pd.DataFrame.
14+
"""
15+
logger.warning(
16+
"This function is not implemented yet. For testing purposes, try mocking the read from db with data from the assets folder."
17+
)

‎scripts/python_training/m3_internal_libraries/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# -*- coding: utf-8 -*-
2+
"""Tasks for the module 3: Internal libraries."""

‎scripts/python_training/m3_internal_libraries/query.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
WITH all_data AS (
2+
SELECT DISTINCT drug_id AS drug_id,
3+
ims_id AS ims_id,
4+
country AS country,
5+
molecule AS molecule
6+
FROM SNOWFLAKE_TABLE_WITH_INFO
7+
)
8+
SELECT drug_id,
9+
ims_id,
10+
country,
11+
molecule
12+
FROM all_data

0 commit comments

Comments
 (0)