11.Light GBM Code
[목적]
- XGBoost Model에서 Feature와 Data를 Handling 하여 처리해주는 LightGBM Model 실습 및 해석
- LightGBM의 경우 Missing Value를 Model 자체 내에서 처리해주기 때문에 삭제하지 않아도 됨
- Big Data를 빠르게 학습함
- 논문에서는 데이터 10,000개 이상일 때 사용하라고 했지만, 일단 돌려보자 -overfitting 땜에..
[Process]
- Define X’s & Y
- Split Train & Valid dataset
- Modeling
- Model 해석
import os
import gc
import re
import pickle
import joblib
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import warnings
warnings.filterwarnings("ignore")
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, f1_score
from lightgbm import LGBMClassifier, LGBMRegressor
from collections import Counter
# Data Loading (수술 時 사망 데이터)
data=pd.read_csv("https://raw.githubusercontent.com/GonieAhn/Data-Science-online-course-from-gonie/main/Data%20Store/example_data.csv")
# Data Quality Checking
col = []
missing = []
level = []
for name in data.columns:
# Missing
missper = data[name].isnull().sum() / data.shape[0]
missing.append(round(missper, 4))
# Leveling
lel = data[name].dropna()
level.append(len(list(set(lel))))
# Columns
col.append(name)
summary = pd.concat([pd.DataFrame(col, columns=['name']),
pd.DataFrame(missing, columns=['Missing Percentage']),
pd.DataFrame(level, columns=['Level'])], axis=1)
drop_col = summary['name'][(summary['Level'] <= 1) | (summary['Missing Percentage'] >= 0.8)]
data.drop(columns=drop_col, inplace=True)
print(">>>> Data Shape : {}".format(data.shape))
drop_col
# X's & Y Split
Y = data['censor']
X = data.drop(columns=['censor'])
idx = list(range(X.shape[0]))
train_idx, valid_idx = train_test_split(idx, test_size=0.3, random_state=2021)
print(">>>> # of Train data : {}".format(len(train_idx)))
print(">>>> # of valid data : {}".format(len(valid_idx)))
print(">>>> # of Train data Y : {}".format(Counter(Y.iloc[train_idx])))
print(">>>> # of valid data Y : {}".format(Counter(Y.iloc[valid_idx])))
[LightGBM Parameters]
- Package : https://lightgbm.readthedocs.io/en/latest/Python-Intro.html
- learning_rate : GBM에서 shrinking 하는 것과 같은 것
- reg_lambda : L2 regularization term on weights (analogous to Ridge regression)
- reg_alpha : L1 regularization term on weight (analogous to Lasso regression)
- objective objective : default = regression, type = enum, options: regression, regression_l1, huber, fair, poisson, quantile, MAPE, gamma, tweedie, binary, multiclass, multiclassova, cross_entropy, cross_entropy_lambda, lambdarank, rank_xendcg, aliases: objective_type, app, application, loss 설명 :objective
- eval_metric [ default according to objective ]
- The metric to be used for validation data.
- The default values are rmse for regression and error for classification.
- Typical values are:
- rmse – root mean square error
- mae – mean absolute error
- logloss – negative log-likelihood
- error – Binary classification error rate (0.5 threshold)
- merror – Multiclass classification error rate
- mlogloss – Multiclass logloss
- auc: Area under the curve [LightGBM]
- Hyperparameter tuning
- n_estimators, learning_rate, max_depth, reg_alpha
- LightGBM은 Hyperparam이 굉장히 많은 알고리즘 중에 하나임
- 위에 4가지만 잘 조정해도 좋은 결과를 얻을 수 있음
# n_estimators
n_tree = [5, 10, 20]
# learning_rate
l_rate = [0.1, 0.3]
# max_depth
m_depth = [3, 5]
# reg_alpha
L1_norm = [0.1, 0.3, 0.5]
# Modeling
save_n = []
save_l = []
save_m = []
save_L1 = []
f1_score_ = []
cnt = 0
for n in n_tree:
for l in l_rate:
for m in m_depth:
for L1 in L1_norm:
print(">>> {} <<<".format(cnt))
cnt +=1
print("n_estimators : {}, learning_rate : {}, max_depth : {}, reg_alpha : {}".format(n, l, m, L1))
model = LGBMClassifier(n_estimators=n, learning_rate=l,
max_depth=m, reg_alpha=L1,
n_jobs=-1, objective='cross_entropy')
model.fit(X.iloc[train_idx], Y.iloc[train_idx])
# Train Acc
y_pre_train = model.predict(X.iloc[train_idx])
cm_train = confusion_matrix(Y.iloc[train_idx], y_pre_train)
print("Train Confusion Matrix")
print(cm_train)
print("Train Acc : {}".format((cm_train[0,0] + cm_train[1,1])/cm_train.sum()))
print("Train F1-Score : {}".format(f1_score(Y.iloc[train_idx], y_pre_train)))
# Test Acc
y_pre_test = model.predict(X.iloc[valid_idx])
cm_test = confusion_matrix(Y.iloc[valid_idx], y_pre_test)
print("Test Confusion Matrix")
print(cm_test)
print("TesT Acc : {}".format((cm_test[0,0] + cm_test[1,1])/cm_test.sum()))
print("Test F1-Score : {}".format(f1_score(Y.iloc[valid_idx], y_pre_test)))
print("-----------------------------------------------------------------------")
print("-----------------------------------------------------------------------")
save_n.append(n)
save_l.append(l)
save_m.append(m)
save_L1.append(L1)
f1_score_.append(f1_score(Y.iloc[valid_idx], y_pre_test))
#joblib.dump(model, './LightGBM_model/Result_{}_{}_{}_{}_{}.pkl'.format(n, l, m, L1, round(save_acc[-1], 4)))
#gc.collect()
-
parameter 4개니까 for loop
print(">>> {} <<<\nBest Test f1-score : {}\nBest n_estimators : {}\nBest Learning Rate : {}\nBest Max_depth : {}\nBest L1-norm : {}".format(np.argmax(f1_score_), f1_score_[np.argmax(f1_score_)], save_n[np.argmax(f1_score_)], save_l[np.argmax(f1_score_)], save_m[np.argmax(f1_score_)], save_L1[np.argmax(f1_score_)]))
best_model = LGBMClassifier(n_estimators=save_n[np.argmax(f1_score_)], learning_rate=save_l[np.argmax(f1_score_)],
max_depth=save_m[np.argmax(f1_score_)], reg_alpha=save_L1[np.argmax(f1_score_)], objective='cross_entropy',
random_state=119)
best_model.fit(X.iloc[train_idx], Y.iloc[train_idx])
# Train Acc
y_pre_train = best_model.predict(X.iloc[train_idx])
cm_train = confusion_matrix(Y.iloc[train_idx], y_pre_train)
print("Train Confusion Matrix")
print(cm_train)
print("Train Acc : {}".format((cm_train[0,0] + cm_train[1,1])/cm_train.sum()))
print("Train F1-Score : {}".format(f1_score(Y.iloc[train_idx], y_pre_train)))
# Test Acc
y_pre_test = best_model.predict(X.iloc[valid_idx])
cm_test = confusion_matrix(Y.iloc[valid_idx], y_pre_test)
print("Test Confusion Matrix")
print(cm_test)
print("TesT Acc : {}".format((cm_test[0,0] + cm_test[1,1])/cm_test.sum()))
print("Test F1-Score : {}".format(f1_score(Y.iloc[valid_idx], y_pre_test)))
feature_map = pd.DataFrame(sorted(zip(best_model.feature_importances_, X.columns), reverse=True), columns=['Score', 'Feature'])
print(feature_map)
# Importance Score Top 10
feature_map_10 = feature_map.iloc[:10]
plt.figure(figsize=(20, 10))
sns.barplot(x="Score", y="Feature", data=feature_map_10.sort_values(by="Score", ascending=False), errwidth=40)
plt.title('XGBoost Importance Features')
plt.tight_layout()
plt.show()
댓글남기기