!pip install kaggle 
from google.colab import files 
files.upload()

#다운로드가 제대로 되었는지 확인한다. 
#ls 명령어는 특정 경로에 어떤 파일이 있는지 확인해 보는 명령어다. 
%ls ~/.kaggle

!kaggle competitions download -c competitive-data-science-predict-future-sales

!unzip competitive-data-science-predict-future-sales.zip

#다운로드가 제대로 되었는지 확인한다. 
#ls 명령어는 특정 경로에 어떤 파일이 있는지 확인해 보는 명령어다. 
%ls ~/.kaggle/competitions/predict-future-sales

import pandas as pd 
import numpy as np 

# TIME SERIES   
from statsmodels.tsa.arima.model import ARIMA, ARIMAResults 
!pip install pmdarima
from pmdarima import auto_arima 
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.api import VAR 
from statsmodels.tsa.stattools import adfuller 
from statsmodels.tsa.statespace.tools import diff 
from statsmodels.tsa.seasonal import seasonal_decompose
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf 
from statsmodels.tools.eval_measures import rmse 

# settings 
import warnings 
warnings.filterwarnings('ignore')

# drop duplicates 
subset = ['date', 'date_block_num', 'shop_id', 'item_id', 'item_cnt_day']
train.duplicated(subset=subset).value_counts()

train.drop_duplicates(subset=subset, inplace=True)
train.shape

# drop shops & items not in test data 
test_shops = test.shop_id.unique()
test_items = test.item_id.unique()

train = train[train.shop_id.isin(test_shops)]
train = train[train.item_id.isin(test_items)]
train.shape

train.sample()

from itertools import product

# create all combinations 
block_shop_combi = pd.DataFrame(list(product(np.arange(34), test_shops)), columns=['date_block_num', 'shop_id'])
shop_item_combi = pd.DataFrame(list(product(test_shops, test_items)), columns=['shop_id', 'item_id'])
all_combi = block_shop_combi.merge(shop_item_combi, on='shop_id', how='inner')
all_combi.head()

# group by monthly 
train_base = all_combi.merge(train, on=['date_block_num', 'shop_id', 'item_id'], how='left')
train_base.head()

train_monthly.head()

#train_monthly = train_monthly.unstack(level=-1).T
#dates=pd.date_range(start = '2013-01-01',end='2015-10-01', freq = 'MS')
#train_monthly.index=dates
# train_monthly = train_monthly.reset_index().melt(id_vars='index')
#train_monthly.head()

train_monthly.head()

train_monthly.date_block_num.max()

test.drop('ID', axis=1)

test = test[['date_block_num', 'shop_id','item_id']]
test.head()

item_grp = item_cat['item_category_name'].apply(lambda x : str(x).split(' ')[0])
item_cat['item_group'] = pd.Categorical(item_grp).codes
item = pd.merge(item, item_cat.loc[:,['item_category_id', 'item_group']], on='item_category_id', how='left')

item.head()

city = shops.shop_name.apply(lambda x : str.replace(x, '!', '')).apply(lambda x : x.split(' ')[0])
shops['city'] = pd.Categorical(city).codes 
shops.head()

train_monthly.head()

test.head()

display(df_train.head())
display(df_test.head()) 

X_train = df_train.drop('item_cnt_month', axis=1)
y_train = df_train['item_cnt_month'] 
X_train.head()

X_test = df_test.copy()
X_test.head()

from sklearn import linear_model, preprocessing 
from sklearn.model_selection import GroupKFold 
import lightgbm as lgb 

params = {'learning_rate' : 0.05,
          'objective' : 'regression',
          'metric' : 'rmse',
          'num_leaves':64, 
          'verbose' : 1,
          'random_state': 42,
          'bagging_fraction':1, # 각각 iteration 에 사용되는 데이터의 일부를 지정하는데, 일반적으로 훈련 속도를 높이거나, 과적합을 피할때 사용한다.
          'feature_fraction':1} # used when your boosting is random forest. 
          # 즉, 랜덤 포레스트 일 경우에 사용. 
          # 0.8 feature fraction은 LightGBM이 Tree를 만들 때 각각 iteration 반복에서 파라미터 중에서 80%를 랜덤으로 선택.

folds = GroupKFold(n_splits=6)
oof_preds = np.zeros(X_train.shape[0])
sub_preds = np.zeros(X_test.shape[0])

for fold_, (trn_, val_) in enumerate(folds.split(X_train, y_train, X_train['date_block_num'])):
  trn_x, trn_y = X_train.iloc[trn_], y_train[trn_]
  val_x, val_y = X_train.iloc[val_], y_train[val_]

  reg = lgb.LGBMRegressor(**params, n_estimators=3000)
  reg.fit(trn_x, trn_y, eval_set=[(val_x, val_y)], early_stopping_rounds=50, verbose=500) 

  oof_preds[val_] = reg.predict(val_x.values, num_iteration=reg.best_iteration_) 
  sub_preds += reg.predict(X_test.values, num_iteration=reg.best_iteration_) / folds.n_splits

print('RMSE:', np.sqrt(mean_squared_error(y_train, oof_preds.clip(0.,20.))))

10122/16100

tx_data.head()

tx_data.describe()

tx_data['Country'].value_counts()

#create a generic user dataframe to keep CustomerID and new segmentation scores 
tx_user = pd.DataFrame(tx_data['CustomerID'].unique())
tx_user.columns = ['CustomerID']
tx_user.head()

#get the max purchase date for each customer and create a dataframe with it
tx_max_purchase = tx_uk.groupby('CustomerID')['InvoiceDate'].max().reset_index()
tx_max_purchase.columns = ['CustomerID', 'MaxPurchaseData']
tx_max_purchase.head()

# Compare the last transaction of dataset with last transaction dates of the individual customer IDs.
tx_max_purchase['Recency'] = (tx_max_purchase['MaxPurchaseData'].max() - tx_max_purchase['MaxPurchaseData']).dt.days 
tx_max_purchase.head()

# merge this dataframe to our new user dataframe 
tx_user = pd.merge(tx_user, tx_max_purchase[['CustomerID', 'Recency']], on='CustomerID')
tx_user.head()

from sklearn.cluster import KMeans 

sse = {} # error 

tx_recency = tx_user[['Recency']]

for k in range(1, 10): 
  kmeans = KMeans(n_clusters=k, max_iter=1000).fit(tx_recency)
  tx_recency['clusters'] = kmeans.labels_ 
  sse[k] = kmeans.inertia_ # 군집의 응집도 값이 작을수록 군집화가 잘 되었다고 평가 

plt.figure()
plt.plot(list(sse.keys()), list(sse.values()))
plt.xlabel("Number of cluster")
plt.show()

tx_user.head()

tx_user.groupby('RecencyCluster')['Recency'].describe()

tx_user = order_cluster('RecencyCluster', 'Recency', tx_user, False)
tx_user

tx_user.groupby('RecencyCluster')['Recency'].describe()

#get order counts for each user and create a dataframe with it 
tx_frequency = tx_uk.groupby('CustomerID')['InvoiceDate'].count().reset_index()
tx_frequency.columns = ['CustomerID', 'Frequency']
tx_frequency.head()

# add this data to our main dataframe 
tx_user = tx_user.merge(tx_frequency, on ='CustomerID')
tx_user.head()

from sklearn.cluster import KMeans 

sse = {} 

tx_frequency = tx_user[['Frequency']]

for k in range(1, 10): 
  kmeans = KMeans(n_clusters=k, max_iter=1000).fit(tx_frequency)  
  tx_frequency['clusters'] = kmeans.labels_ 
  sse[k] = kmeans.inertia_ 

plt.figure()
plt.plot(list(sse.keys()), list(sse.values()))
plt.xlabel('Number of cluster')
plt.show()

tx_user.groupby('FrequencyCluster')['Frequency'].describe()

# order the frequency cluster 
tx_user = order_cluster('FrequencyCluster', 'Frequency', tx_user, True)
tx_user.groupby('FrequencyCluster')['Frequency'].describe()

# calculate revenue for each customer 
tx_uk['Revenue'] = tx_uk['UnitPrice'] * tx_uk['Quantity']
tx_revenue = tx_uk.groupby('CustomerID').Revenue.sum().reset_index()
tx_revenue.head()

# merge it with our main dataframe 
tx_user = tx_user.merge(tx_revenue, on='CustomerID')
tx_user.head()

from sklearn.cluster import KMeans

sse={} # error

tx_monetary = tx_user[['Revenue']]
for k in range(1, 10):
    kmeans = KMeans(n_clusters=k, max_iter=1000).fit(tx_monetary)
    tx_monetary["clusters"] = kmeans.labels_ 
    sse[k] = kmeans.inertia_ 

plt.figure()
plt.plot(list(sse.keys()), list(sse.values()))
plt.xlabel("Number of cluster")
plt.show()

#apply clustering
kmeans = KMeans(n_clusters=4)
tx_user['RevenueCluster'] = kmeans.fit_predict(tx_user[['Revenue']])

#order the cluster numbers
tx_user = order_cluster('RevenueCluster', 'Revenue', tx_user, True)

#show details of the dataframe
tx_user.groupby('RevenueCluster')['Revenue'].describe()

# calculate overall score and use mean() to see details 

tx_user['OverallScore'] = tx_user['RecencyCluster'] + tx_user['FrequencyCluster'] + tx_user['RevenueCluster']
tx_user.groupby('OverallScore')['Recency', 'Frequency', 'Revenue'].mean()

tx_uk['InvoiceDate'].describe()

tx_uk.InvoiceDate = pd.to_datetime(tx_uk.InvoiceDate).dt.date
tx_3m = tx_uk[(tx_uk.InvoiceDate < date(2011, 6, 1)) & (tx_uk.InvoiceDate >= date(2011, 3, 1))].reset_index()
tx_3m.InvoiceDate # 3 months time 

tx_uk.InvoiceDate = pd.to_datetime(tx_uk.InvoiceDate).dt.date
tx_6m = tx_uk[(tx_uk.InvoiceDate >= date(2011, 6, 1)) & (tx_uk.InvoiceDate < date(2011, 12, 1))].reset_index()
tx_6m.InvoiceDate # 6 months time 

tx_user_6m

# plot LTV histogram 
sns.distplot(tx_user_6m['m6_Revenue'], kde=False)
plt.title('6m Revenue');

tx_merge = tx_user.merge(tx_user_6m, on='CustomerID', how='left')
tx_merge = tx_merge.fillna(0)  #Only people who are in the timeline of tx_user_6m
tx_merge.head()

tx_graph = tx_merge.query('m6_Revenue < 50000') #because max values are ending at 50,000 as seen in graph above

sns.scatterplot(data=tx_graph.query('Segment == "Low-Value"'), x='OverallScore', y='m6_Revenue', label='Low')
sns.scatterplot(data=tx_graph.query('Segment == "Mid-Value"'), x='OverallScore', y='m6_Revenue', label='Mid')
sns.scatterplot(data=tx_graph.query('Segment == "High-Value"'), x='OverallScore', y='m6_Revenue', label='High')

plt.title('LTV')
plt.xlabel('RFM Score')
plt.ylabel('6m LTV')
plt.legend(bbox_to_anchor=(1.02, 1.02));

tx_merge.head()

# creating 3 clusters 
kmeans = KMeans(n_clusters=3) 
tx_merge['LTVCluster'] = kmeans.fit_predict(tx_merge[['m6_Revenue']])
tx_merge.head()

tx_cluster.groupby('LTVCluster')['m6_Revenue'].describe()

tx_cluster.head()

# convert categorical columns to numerical 
tx_class = pd.get_dummies(tx_cluster)
tx_class.head()

# calculate and show correlations 
corr_matrix = tx_class.corr()
corr_matrix['LTVCluster'].sort_values(ascending=False)

#XGBoost Multiclassification Model 
ltv_xgb_model = xgb.XGBClassifier(max_depth=5, learning_rate=0.1, n_jobs=-1)
ltv_xgb_model.fit(X_train, y_train) 

print('Accuracy of XGB classfier on training set: {:.2f}'.format(ltv_xgb_model.score(X_train, y_train)))
print('Accuracy of XGB classfier on test set: {:.2f}'.format(ltv_xgb_model.score(X_test, y_test)))

y_pred = ltv_xgb_model.predict(X_test)

print(classification_report(y_test, y_pred))

y_test

y_pred

!pip install kaggle 
from google.colab import files 
files.upload()

!mkdir -p ~/.kaggle #create folder name Kaggle
!cp kaggle.json ~/.kaggle #copy kaggle.jason into folder Kaggle
!chmod 600 ~/.kaggle/kaggle.json #ignore Permission Warning 

%ls ~/.kaggle

!kaggle competitions download -c walmart-recruiting-store-sales-forecasting

!unzip walmart-recruiting-store-sales-forecasting.zip

!unzip features.csv.zip    
!unzip sampleSubmission.csv.zip  
!unzip test.csv.zip
!unzip train.csv.zip

!mkdir -p ~/.kaggle/competitions/walmart-recruiting-store-sales-forecasting #create folder name Kaggle
!cp features.csv ~/.kaggle/competitions/walmart-recruiting-store-sales-forecasting #copy into folder Kaggle 
!cp sampleSubmission.csv  ~/.kaggle/competitions/walmart-recruiting-store-sales-forecasting
!cp stores.csv  ~/.kaggle/competitions/walmart-recruiting-store-sales-forecasting
!cp test.csv  ~/.kaggle/competitions/walmart-recruiting-store-sales-forecasting
!cp train.csv  ~/.kaggle/competitions/walmart-recruiting-store-sales-forecasting

%ls ~/.kaggle/competitions/walmart-recruiting-store-sales-forecasting

print(train_df.shape)
print(test_df.shape)

train_df.describe().T

train.sample()

feature_store.sample()

sns.lineplot(data=df_weeks, x=df_weeks.index, y='Weekly_Sales')
plt.title('Sales over weeks')

sns.lineplot(x = df_weeks.index, y=df_weeks['MarkDown1'], label='MarkDown1')
sns.lineplot(x = df_weeks.index, y=df_weeks['MarkDown2'], label='MarkDown2')
sns.lineplot(x = df_weeks.index, y=df_weeks['MarkDown3'], label='MarkDown3')
sns.lineplot(x = df_weeks.index, y=df_weeks['MarkDown4'], label='MarkDown4')
sns.lineplot(x = df_weeks.index, y=df_weeks['MarkDown5'], label='MarkDown5')
sns.lineplot(data=df_weeks, x=df_weeks.index, y='Weekly_Sales', label='Weekly Sales') 

sns.lineplot(data=weekly_sales2010, x=weekly_sales2010['Weekly_Sales']['mean'].index, y=weekly_sales2010['Weekly_Sales']['mean'])
sns.lineplot(data=weekly_sales2011, x=weekly_sales2011['Weekly_Sales']['mean'].index, y=weekly_sales2011['Weekly_Sales']['mean'])
sns.lineplot(data=weekly_sales2012, x=weekly_sales2012['Weekly_Sales']['mean'].index, y=weekly_sales2012['Weekly_Sales']['mean'])

print(train_df.shape)
print(train_plt.shape)

fig = plt.figure(figsize=(14, 5))
ax1 = fig.add_subplot(1, 2, 1) # 서브플롯들을 2 x 1 배열로 배치 그중 첫번째
ax2 = fig.add_subplot(1, 2, 2)  # 서브플롯들을 2 x 1 배열로 배치 그중 두번째

# Temperature
sns.regplot(data=train_plt, x='Temperature', y='Weekly_Sales', ax=ax1) 
ax1.set_title('Temparature and Sales by holiday')

sns.histplot(data=train_plt, x='Temperature', hue='IsHoliday', ax=ax2) 
ax2.set_title('Temparature and Sales by holiday')

plt.show();

fig = plt.figure(figsize=(14, 5))
ax1 = fig.add_subplot(1, 2, 1) # 서브플롯들을 2 x 1 배열로 배치 그중 첫번째
ax2 = fig.add_subplot(1, 2, 2)  # 서브플롯들을 2 x 1 배열로 배치 그중 두번째

# Fuel_Price
sns.regplot(data=train_plt, x='Fuel_Price', y='Weekly_Sales', ax=ax1) 
ax1.set_title('Fuel_Price and Sales by holiday')

sns.histplot(data=train_plt, x='Fuel_Price', hue='IsHoliday', ax=ax2) 
ax2.set_title('Fuel_Price and Sales by holiday')

plt.show();

fig = plt.figure(figsize=(14, 5))
ax1 = fig.add_subplot(1, 2, 1) # 서브플롯들을 2 x 1 배열로 배치 그중 첫번째
ax2 = fig.add_subplot(1, 2, 2)  # 서브플롯들을 2 x 1 배열로 배치 그중 두번째

# CPI
sns.regplot(data=train_plt, x='CPI', y='Weekly_Sales', ax=ax1) 
ax1.set_title('CPI and Sales by holiday')

sns.histplot(data=train_plt, x='CPI', hue='IsHoliday', ax=ax2) 
ax2.set_title('CPI and Sales by holiday')

plt.show();

fig = plt.figure(figsize=(14, 5))
ax1 = fig.add_subplot(1, 2, 1) # 서브플롯들을 2 x 1 배열로 배치 그중 첫번째
ax2 = fig.add_subplot(1, 2, 2)  # 서브플롯들을 2 x 1 배열로 배치 그중 두번째

# Unemployment
sns.regplot(data=train_plt, x='Unemployment', y='Weekly_Sales', ax=ax1) 
ax1.set_title('Unemployment and Sales by holiday')

sns.histplot(data=train_plt, x='Unemployment', hue='IsHoliday', ax=ax2) 
ax2.set_title('Unemployment and Sales by holiday')

plt.show();

sizes = train_plt.groupby('Size')['Weekly_Sales'].mean().reset_index()

sns.lineplot(data=sizes, x=sizes.index, y=sizes.Weekly_Sales) 
plt.title('Store size and Sales');

store_type = stores[['Type', 'Size']]
sns.boxplot(data=store_type, x='Type', y='Size')

dept = train_plt.groupby('Dept')['Weekly_Sales'].mean().reset_index() 

sns.barplot(data=dept, x='Dept', y='Weekly_Sales')
plt.title('Department and sales');

plt.figure(figsize=(16, 16))

sns.heatmap(train_df.corr(), 
            linewidths=0.1,
            square=True,
            annot=True,
            fmt='.2f',
            cmap='Blues')

weekly_sales_corr = pd.DataFrame(train_df.corr()['Weekly_Sales']).sort_values(by='Weekly_Sales', ascending=False)

plt.figure(figsize=(8,4))
sns.barplot(data=weekly_sales_corr, x=weekly_sales_corr.index, y='Weekly_Sales')
plt.xticks(rotation=70)
plt.title('Feature Correlation with Sales');

data_train.isna().sum()[data_train.isna().sum() > 0]

data_test.isna().sum()[data_test.isna().sum() > 0]

feat_model = xgb.XGBRegressor(random_state=0, objective='reg:squarederror')
feat_model.fit(X_train, y_train)

!pip install eli5
import eli5
from eli5.sklearn import PermutationImportance 

perm = PermutationImportance(feat_model, random_state=1)
perm.fit(X_valid, y_valid)

feature_weights = eli5.show_weights(perm, top=len(X_train.columns), feature_names= X_valid.columns.tolist())

feature_weights