nba_player_of_the_week_df.sample()

nba_player_of_the_week_df.describe()

height_weight_age_df.sample()

# min-max-normalization 
scaler = preprocessing.MinMaxScaler()
normalized_data = scaler.fit_transform(height_weight_age_df)
normalized_data

normalized_df.describe()

df = pd.DataFrame(np.array([25, 49, 32, 35, 40]), columns=['Age'])
df 

normalized_data = scaler.fit_transform(df)
normalized_data 

normalized_df = pd.DataFrame(normalized_data, columns=['Age'])
normalized_df

df

normalized_data = scaler.fit_transform(df)
normalized_data

normalized_df = pd.DataFrame(normalized_data, columns=['Salary'])
normalized_df

titanic_df = pd.read_csv('/content/drive/MyDrive/data/titanic.csv')
titanic_df.sample()

titanic_sex_embarked = titanic_df[['Sex', 'Embarked']]
titanic_sex_embarked.head()

one_hot_encoded_df = pd.get_dummies(titanic_sex_embarked)
one_hot_encoded_df.head() 

one_hot_encoded_df = pd.get_dummies(data=titanic_df, columns=['Sex', 'Embarked'])
one_hot_encoded_df.head()

boston_df.info()

fig, axs = plt.subplots(figsize=(16,8), ncols=4, nrows=2)

features = ['RM', 'ZN', 'INDUS', 'NOX', 'AGE', 'PTRATIO', 'LSTAT','RAD']

for i, feature in enumerate(features):
    row = int(i/4)
    col = i % 4  
    sns.regplot(x=feature, y='Price', data=boston_df, ax=axs[row][col])

lr = LinearRegression()
lr.fit(X_train, y_train)

print('training set에서의 성능')
y_train_predict = lr.predict(X_train)
mse = mean_squared_error(y_train, y_train_predict)
print('rmse: {0:.3f}'.format(np.sqrt(mse)))

print('test set에서의 성능')
y_test_predict = lr.predict(X_test)
mse = mean_squared_error(y_test, y_test_predict)
print('rmse: {0:.3f}'.format(np.sqrt(mse)))

# 다항변형기로 데이터를 다항회귀를 위해 가공한다  
polynomial_transformer = PolynomialFeatures(2)
polynomial_data = polynomial_transformer.fit_transform(boston.data)

print(polynomial_data.shape)
polynomial_data

lr = LinearRegression()
lr.fit(X_train, y_train)

print('training set에서의 성능')
y_train_predict = lr.predict(X_train)
mse = mean_squared_error(y_train, y_train_predict)
print('rmse: {0:.3f}'.format(np.sqrt(mse)))

print('test set에서의 성능')
y_test_predict = lr.predict(X_test)
mse = mean_squared_error(y_test, y_test_predict)
print('rmse: {0:.3f}'.format(np.sqrt(mse)))

rf = RandomForestRegressor(random_state=0, n_estimators=1000)

neg_mse_scores = cross_val_score(rf, X, y, scoring='neg_mean_squared_error', cv=5)
rmse_scores = np.sqrt(-1 * neg_mse_scores)
avg_rmse = np.mean(rmse_scores)
avg_rmse

from sklearn.tree import DecisionTreeRegressor 
from sklearn.ensemble import GradientBoostingRegressor 
from xgboost import XGBRegressor 
from lightgbm import LGBMRegressor 

dt_reg = DecisionTreeRegressor(random_state=0, max_depth=4)
rf_reg = RandomForestRegressor(random_state=0, n_estimators=1000)
gb_reg = GradientBoostingRegressor(random_state=0, n_estimators=1000)
xgb_reg = XGBRegressor(n_estimators=1000)
lgb_reg =LGBMRegressor(n_estimators=1000)

models = [dt_reg, rf_reg, gb_reg, xgb_reg, lgb_reg]

for model in models: 
    get_rmse(model, X, y)

import seaborn as sns 

rf_reg = RandomForestRegressor(n_estimators=1000)

rf_reg.fit(X, y)

s = pd.Series(data=rf_reg.feature_importances_, index=X.columns).sort_values(ascending=False)
sns.barplot(x=s, y=s.index)

!pip install kaggle 
from google.colab import files 
files.upload()

ls -1ha kaggle.json

#다운로드가 제대로 되었는지 확인한다. 
#ls 명령어는 특정 경로에 어떤 파일이 있는지 확인해 보는 명령어다. 
%ls ~/.kaggle

#Copy API command 후 데이터셋 다운로드하기  
!kaggle datasets download -d kaggle/kaggle-survey-2017

#파일 압축 풀기
!unzip kaggle-survey-2017.zip

#다운로드가 제대로 되었는지 확인한다. 
#ls 명령어는 특정 경로에 어떤 파일이 있는지 확인해 보는 명령어다. 
%ls ~/.kaggle/competitions/kaggle-survey-2017

!pip install kaggle 
from google.colab import files 
files.upload()

ls -1ha kaggle.json

#대회 약관 동의하고, 데이터셋 불러오기 
!kaggle competitions download -c bike-sharing-demand
!unzip bike-sharing-demand.zip

#ls 명령어는 특정 경로에 어떤 파일이 있는지 확인해 보는 명령어다. 
%ls ~/.kaggle/competitions/bike-sharing-demand 

train.shape

train.info()

train.head()

train.temp.describe()

train.isnull().sum()

figure, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(nrows=2, ncols=3)
figure.set_size_inches(18,8)

sns.barplot(data=train, x='year', y='count', ax=ax1)
sns.barplot(data=train, x='month', y='count', ax=ax2)
sns.barplot(data=train, x='day', y='count', ax=ax3)
sns.barplot(data=train, x='hour', y='count', ax=ax4)
sns.barplot(data=train, x='minute', y='count', ax=ax5)
sns.barplot(data=train, x='second', y='count', ax=ax6)

ax1.set(ylabel='Count', title='연도별 대여량')
ax2.set(ylabel='month', title='월별 대여량')
ax3.set(ylabel='day', title='일별 대여량')
ax4.set(ylabel='hour', title='시간별 대여량')

fig, axes = plt.subplots(nrows=2, ncols=2)
fig.set_size_inches(12, 10)

sns.boxplot(data=train, y='count', orient='v', ax=axes[0][0])
sns.boxplot(data=train, y='count', x='season', orient='v', ax=axes[0][1])
sns.boxplot(data=train, y='count', x='hour', orient='v', ax=axes[1][0])
sns.boxplot(data=train, y='count', x='workingday', orient='v', ax=axes[1][1])

axes[0][0].set(ylabel='Count', title='대여량')
axes[0][1].set(xlabel='Season', ylabel='Count', title='계절별 대여량')
axes[1][0].set(xlabel='Hour Of The Day', ylabel='Count', title='시간별 대여량')
axes[1][1].set(xlabel='Working Day', ylabel='Count', title='근무일 여부에 따른 대여량')

train['dayofweek'] = train['datetime'].dt.dayofweek
train['dayofweek'].value_counts()

fig, (ax1, ax2, ax3, ax4, ax5) = plt.subplots(nrows=5)
fig.set_size_inches(18,25)

sns.pointplot(data=train, x='hour', y='count', ax=ax1)

sns.pointplot(data=train, x='hour', y='count', hue='workingday', ax=ax2)

sns.pointplot(data=train, x='hour', y='count', hue='dayofweek', ax=ax3)

sns.pointplot(data=train, x='hour', y='count', hue='weather', ax=ax4)

sns.pointplot(data=train, x='hour', y='count', hue='season', ax=ax5)

corrMatt = train[['temp', 'atemp', 'casual', 'registered', 'humidity', 'windspeed', 'count']]
corrMatt = corrMatt.corr() #상관관계 
print(corrMatt)

#삼각형 마스크를 만든다(위 쪽 삼각형에 True, 아래 삼각형에 False)
mask = np.array(corrMatt)
mask[np.tril_indices_from(mask)] = False 

#상관관계 분석 시각화 

fig, ax = plt.subplots()
fig.set_size_inches(20,10)
sns.heatmap(corrMatt, 
            mask=mask, #표시하지 않을 마스크 부분을 지정한다.   
            vmax=.8, #컬러바 범위  
            square=True, #정사각형 
            annot=True)

fig, (ax1, ax2, ax3) = plt.subplots(ncols=3)
fig.set_size_inches(12, 5)
sns.regplot(x='temp', y='count', data=train, ax=ax1) #온도
sns.regplot(x='windspeed', y='count', data=train, ax=ax2) #풍속
sns.regplot(x='humidity', y='count', data=train, ax=ax3) #습도 

def concatenate_year_month(datetime):
    return '{0}-{1}'.format(datetime.year, datetime.month)

train['year_month'] = train['datetime'].apply(concatenate_year_month)

train[['datetime', 'year_month']].head()

fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)
fig.set_size_inches(18, 4)

sns.barplot(data=train, x='year', y='count', ax=ax1)
sns.barplot(data=train, x='month', y='count', ax=ax2)

fig, ax3 = plt.subplots(nrows=1, ncols=1)
fig.set_size_inches(18, 4)

sns.barplot(data=train, x='year_month', y='count', ax=ax3)

# count값의 데이터 분포도를 파악 

figure, axes = plt.subplots(ncols=2, nrows=2)
figure.set_size_inches(12, 10)

sns.distplot(train['count'], ax=axes[0][0])
stats.probplot(train['count'], dist='norm', fit=True, plot=axes[0][1])

!pip install kaggle 
from google.colab import files 
files.upload()

ls -1ha kaggle.json

#ls 명령어는 특정 경로에 어떤 파일이 있는지 확인해 보는 명령어다. 
%ls ~/.kaggle

!kaggle datasets download -d kaggle/kaggle-survey-2017
!unzip kaggle-survey-2017.zip

#ls 명령어는 특정 경로에 어떤 파일이 있는지 확인해 보는 명령어다. 
%ls ~/.kaggle/competitions/kaggle-survey-2017

mcr.sample()

#성별 
#countplot 명령을 사용하면 각 카테고리 값별로 데이터가 얼마나 있는지 표시할 수 있다.
sns.countplot(y='GenderSelect', data=mcr)

#학력 
sns.countplot(y='FormalEducation', data=mcr)

sns.countplot(y='FormalEducation',
              hue='GenderSelect',
              data=mcr).legend(loc='center left',
                       bbox_to_anchor=(1, 0.5))

#Seaborn의 distplot 명령은 러그와 커널 밀도 표시 기능이 있어서 Matplotlib의 hist 명령보다 많이 사용된다.

sns.distplot(mcr[mcr['Age']>0]['Age'])

#연령대를 성별에 따라 그리기 
figure, (ax1, ax2) = plt.subplots(ncols=2)
figure.set_size_inches(12, 5)

sns.distplot(mcr[mcr['GenderSelect'] == 'Female']['Age'].dropna(),
             norm_hist=False, ax=ax1)
plt.title('Female')

sns.distplot(mcr[mcr['GenderSelect'] == 'Male']['Age'].dropna(),
             norm_hist=False, ax=ax2)
plt.title('Male')

netflix_titles.sample()

df = netflix_titles.groupby(['type','release_year'])['title'].nunique().reset_index()
df = df[df['release_year']>=2011]
df.sample()

sns.lineplot(data=df, x='release_year', y='title', ci=None)

df.sample()

sns.lineplot(data=df, x='release_year', y='title', ci=None)
plt.axhline(df['title'].median(), linestyle='dashed')

sns.lineplot(data=df, x='release_year', y='title', ci=None, hue='type')

sns.lineplot(data=df, x='release_year', y='title', ci=None, hue='type', style='type')

sns.lineplot(data=df, x='release_year', y='title', ci=None, hue='type', style='type', markers=True)

sns.displot(x=df['title'])

sns.displot(x=df['title'], kde=True)

sns.displot(x=df['title'], kind='kde')

sns.displot(data=df, x='title', hue='type')

sns.displot(data=df, x='title', hue='type', kind='kde')

sns.displot(data=df, x='title', hue='type', col='release_year', col_wrap=4)

df.sample()

sns.boxplot(data=df, y='title')

Q1 = df['title'].quantile(q=0.25)
Q3 = df['title'].quantile(q=0.75)
IQR = Q3 - Q1
IQR

IQR_df = df[(df['title'] <= (Q3 + IQR *1.5)) & (df['title'] >= (Q1 - IQR *1.5))]
len(IQR_df)

sns.boxplot(data=IQR_df, y='title')

heatmap = netflix_titles.pivot_table(index='year', columns='rating', values='title', aggfunc='count')
heatmap

plt.figure(figsize=(12, 6))
sns.heatmap(heatmap, annot=True, fmt='.0f', cmap='Blues')

df['type'].unique()

#top bar 
total = df.groupby('release_year').sum().reset_index()

# bar chart 1 -> top bars (group of 'type != Movie')
bar1 = sns.barplot(x="release_year",  y="title", data=total, color='darkblue')

# bottom bar ->  take only 'type == Movie' values from the data
movie = df[df.type=='Movie']

# bar chart 2 -> bottom bars (group of 'type == Movie')
bar2 = sns.barplot(x="release_year", y="title", data=movie, estimator=sum, ci=None,  color='lightblue')

# add legend
top_bar = mpatches.Patch(color='darkblue', label='TV Show')
bottom_bar = mpatches.Patch(color='lightblue', label='Movie')
plt.legend(handles=[top_bar, bottom_bar])

# show the graph
plt.show()

# from raw value to percentage
total = df.groupby('release_year')['title'].sum().reset_index()
movie = df[df.type=='Movie'].groupby('release_year')['title'].sum().reset_index()
movie['title'] = [i / j * 100 for i,j in zip(movie['title'], total['title'])]
total['title'] = [i / j * 100 for i,j in zip(total['title'], total['title'])]

# bar chart 1 -> top bars (group of 'type != Movie')
bar1 = sns.barplot(x="release_year",  y="title", data=total, color='darkblue')

# bar chart 2 -> bottom bars (group of 'type == Movie')
bar2 = sns.barplot(x="release_year", y="title", data=movie, color='lightblue')

# add legend
top_bar = mpatches.Patch(color='darkblue', label='TV Show')
bottom_bar = mpatches.Patch(color='lightblue', label='Movie')
plt.legend(handles=[top_bar, bottom_bar])

# show the graph
plt.show()

total = df.groupby('release_year')['title'].sum().reset_index()
movie = df[df.type=='Movie'].groupby('release_year')['title'].sum().reset_index()
total

movie

netflix_titles['ym'] = pd.to_datetime(netflix_titles['date_added']).dt.strftime('%Y-%m')
netflix_titles['ym'] 

group = netflix_titles.groupby(['type','ym'])['title'].count().reset_index().copy()

#2019년 10월 이후 데이터만 출력 
group = group[group['ym']>='2019-10']

group.head()

#set_index()로 멀티 인덱스 지정하기 

group = group.set_index(['ym','type'])
group.head()

#unstack할 컬럼의 레벨 지정하기 

group = group.unstack(1)
group.head()

group.plot(kind='bar',stacked=True, color=['skyblue', 'orange', 'yellow'], figsize=(12,6))
plt.xticks(rotation=70)
plt.legend(bbox_to_anchor=(1.02, 1), loc=2, borderaxespad=0.) #legend 위치 변경 

netflix_type = netflix_titles['type'].value_counts()
netflix_type

import matplotlib.pyplot as plt
plt.pie(netflix_type, labels = netflix_type.index,
        autopct='%1.1f%%', shadow=True, startangle=140)
plt.axis=('equal')
plt.title('Type of Netflix Titles')
plt.show()

netflix_titles.sample()

netflix_titles['date_added'].dtype

netflix_titles['date'] = pd.to_datetime(netflix_titles['date_added'])
netflix_titles['date'] 

netflix_titles['ym'] = pd.to_datetime(netflix_titles['date_added']).dt.strftime('%Y-%m')
netflix_titles['ym']

netflix_titles['ym_dt'] = pd.to_datetime(netflix_titles['ym'], format='%Y-%m')
netflix_titles['ym_dt']

netflix_titles['date'].dt.to_period('M')

netflix_titles['date'].dt.to_period('Q')

netflix_titles['date'].dt.to_period('M').astype(int)

netflix_titles['date'].dt.to_period('Q').astype(int)

pd.to_datetime(netflix_titles['ym'], format='%Y-%m').astype(int)

netflix_titles['date'].dt.to_period('Q').dt.strftime('%Y-%q') 

netflix_titles['date']

netflix_titles['dayname'] = netflix_titles['date'].dt.day_name()
netflix_titles['dayname'] 

dayname = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']

pd.Categorical(netflix_titles['dayname'], categories=dayname, ordered=True)

netflix_titles['date']

netflix_titles[['date','previous_date']]

growth = netflix_titles.groupby(['date'])['title'].nunique().reset_index()
growth.sample()

growth['previous_date'] = growth['date'].apply(lambda x : x - relativedelta(years=1))
growth.sample()

growth = growth.merge(growth, how='left', left_on='previous_date', right_on='date')
growth.sample()

growth = growth[['date_x', 'title_x', 'previous_date_x', 'title_y']].rename({'date_x': 'date', 'title_x':'contents', 'previous_date_x':'previous_date', 'title_y':'previous_contents'}, axis = 1)
growth.sample()

growth['yoy'] = (growth['contents']-growth['previous_contents'])/growth['previous_contents']
growth.tail()

au = growth[['date', 'contents']].rename({'contents':'dau'}, axis = 1)
au.sample()

au.head(30)

netflix_titles.sample()

#value_counts를 사용하면 그룹화 된 데이터의 카운트 값을 보여준다.
type_count = pd.DataFrame(netflix_titles['type'].value_counts())
type_count

#normalize=True 옵션을 사용하면, 해당 데이터가 전체 데이터에서 어느 정도의 비율을 차지하는 지 알 수 있다.  
type_percent = pd.DataFrame(netflix_titles['type'].value_counts(normalize=True))
type_percent

df = type_count.merge(type_percent, how='left', left_index=True, right_index=True)

df.columns = ['콘텐츠 수', '비율']

df.head()

netflix_titles.groupby('release_year')['title'].count()

netflix_titles.groupby('release_year')['title'].count().reset_index()

group = netflix_titles.groupby('release_year').agg(title_cnt = ('title', 'count'), country_cnt = ('country', 'count'))
group

#release_yeaer가 2000년 이하인 인덱스 

group[group.index < 2000].index

group.drop(group[group.index < 2000].index, inplace=True)
group

group.drop(['country_cnt'], axis=1, inplace=True)
group

#멀티 인덱스 생성 
multi_index = netflix_titles.groupby(['release_year', 'type'])['title'].count().reset_index().set_index(['release_year', 'type'])
multi_index

#멀티 인덱스로 2000년 release 된 title을 선택하고 싶은 경우  

multi_index.xs(2000, level='release_year')

#멀티 인덱스로 type이 Movie인 title을 선택하고 싶은 경우 

multi_index.xs('Movie', level='type')

#duration 전처리 
netflix_titles['duration'] = pd.to_numeric(netflix_titles['duration'].str.split('Season|min', expand=True)[0].replace({'NaN':None}))
netflix_titles['duration']

group = netflix_titles.groupby(['type','rating'])[['release_year', 'duration']].agg(['min', 'mean'])
group.head()

group.stack(level=0).head()

group.stack(level=1).head()

stacked = group.stack()
stacked.head()

group['release_year'].stack().head()