LightGBM
1.读取csv数据并指定参数建模
1# coding: utf-8
2import json
3import lightgbm as lgb
4import pandas as pd
5from sklearn.metrics import mean_squared_error
6
7# 加载数据
8print('Load data...')
9df_train = pd.read_csv('./data/regression.train.txt', header=None, sep='\t')
10df_test = pd.read_csv('./data/regression.test.txt', header=None, sep='\t')
11
12# 设定训练集和测试集
13y_train = df_train[0].values
14y_test = df_test[0].values
15X_train = df_train.drop(0, axis=1).values
16X_test = df_test.drop(0, axis=1).values
17
18# 构建lgb中的Dataset格式，和xgboost中的DMatrix是对应的
19lgb_train = lgb.Dataset(X_train, y_train)
20lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
21
22# 参数
23params = {
24 'task': 'train',
25 'boosting_type': 'gbdt',
26 'objective': 'regression',
27 'metric': {'l2', 'auc'},
28 'num_leaves': 31,
29 'learning_rate': 0.05,
30 'feature_fraction': 0.9,
31 'bagging_fraction': 0.8,
32 'bagging_freq': 5,
33 'verbose': 0
34}
35
36print('开始训练...')
37# 训练
38gbm = lgb.train(params,
39 lgb_train,
40 num_boost_round=20,
41 valid_sets=lgb_eval,
42 early_stopping_rounds=5)
43
44# 保存模型
45print('保存模型...')
46# 保存模型到文件中
47gbm.save_model('model.txt')
48
49print('开始预测...')
50# 预测
51y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)
52# 评估
53print('预估结果的rmse为:')
54print(mean_squared_error(y_test, y_pred) ** 0.5)
55
56
57Load data...
58开始训练...
59[1] valid_0's auc: 0.764496 valid_0's l2: 0.24288
60Training until validation scores don't improve for 5 rounds.
61[2] valid_0's auc: 0.766173 valid_0's l2: 0.239307
62[3] valid_0's auc: 0.785547 valid_0's l2: 0.235559
63[4] valid_0's auc: 0.797786 valid_0's l2: 0.230771
64[5] valid_0's auc: 0.805155 valid_0's l2: 0.226297
65[6] valid_0's auc: 0.803083 valid_0's l2: 0.22359
66[7] valid_0's auc: 0.809622 valid_0's l2: 0.220982
67[8] valid_0's auc: 0.808114 valid_0's l2: 0.218316
68[9] valid_0's auc: 0.805671 valid_0's l2: 0.215884
69[10] valid_0's auc: 0.805365 valid_0's l2: 0.213232
70[11] valid_0's auc: 0.804857 valid_0's l2: 0.211087
71[12] valid_0's auc: 0.805453 valid_0's l2: 0.20914
72Early stopping, best iteration is:
73[7] valid_0's auc: 0.809622 valid_0's l2: 0.220982
74保存模型...
75开始预测...
76预估结果的rmse为:
770.4700869286041175
2.添加样本权重训练
1# coding: utf-8
2import json
3import lightgbm as lgb
4import pandas as pd
5import numpy as np
6from sklearn.metrics import mean_squared_error
7import warnings
8warnings.filterwarnings("ignore")
9
10# 加载数据集
11print('加载数据...')
12df_train = pd.read_csv('./data/binary.train', header=None, sep='\t')
13df_test = pd.read_csv('./data/binary.test', header=None, sep='\t')
14W_train = pd.read_csv('./data/binary.train.weight', header=None)[0]
15W_test = pd.read_csv('./data/binary.test.weight', header=None)[0]
16
17y_train = df_train[0].values
18y_test = df_test[0].values
19X_train = df_train.drop(0, axis=1).values
20X_test = df_test.drop(0, axis=1).values
21
22num_train, num_feature = X_train.shape
23
24# 加载数据的同时加载权重
25lgb_train = lgb.Dataset(X_train, y_train,
26 weight=W_train, free_raw_data=False)
27lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train,
28 weight=W_test, free_raw_data=False)
29
30# 设定参数
31params = {
32 'boosting_type': 'gbdt',
33 'objective': 'binary',
34 'metric': 'binary_logloss',
35 'num_leaves': 31,
36 'learning_rate': 0.05,
37 'feature_fraction': 0.9,
38 'bagging_fraction': 0.8,
39 'bagging_freq': 5,
40 'verbose': 0
41}
42
43# 产出特征名称
44feature_name = ['feature_' + str(col) for col in range(num_feature)]
45
46print('开始训练...')
47gbm = lgb.train(params,
48 lgb_train,
49 num_boost_round=10,
50 valid_sets=lgb_train, # 评估训练集
51 feature_name=feature_name,
52 categorical_feature=[21])
53
54
55加载数据...
56开始训练...
57[1] training's binary_logloss: 0.680298
58[2] training's binary_logloss: 0.672021
59[3] training's binary_logloss: 0.664444
60[4] training's binary_logloss: 0.655536
61[5] training's binary_logloss: 0.647375
62[6] training's binary_logloss: 0.64095
63[7] training's binary_logloss: 0.63514
64[8] training's binary_logloss: 0.628769
65[9] training's binary_logloss: 0.622774
66[10] training's binary_logloss: 0.616895
3.模型的载入与预测
1# 查看特征名称
2print('完成10轮训练...')
3print('第7个特征为:')
4print(repr(lgb_train.feature_name[6]))
5
6# 存储模型
7gbm.save_model('./model/lgb_model.txt')
8
9# 特征名称
10print('特征名称:')
11print(gbm.feature_name())
12
13# 特征重要度
14print('特征重要度:')
15print(list(gbm.feature_importance()))
16
17# lgb.Booster加载模型
18print('加载模型用于预测')
19bst = lgb.Booster(model_file='./model/lgb_model.txt')
20
21# 预测
22y_pred = bst.predict(X_test)
23
24# 在测试集评估效果
25print('在测试集上的rmse为:')
26print(mean_squared_error(y_test, y_pred) ** 0.5)
27
28
29完成10轮训练...
30第7个特征为:
31'feature_6'
32特征名称:
33['feature_0', 'feature_1', 'feature_2', 'feature_3', 'feature_4', 'feature_5', 'feature_6', 'feature_7', 'feature_8', 'feature_9', 'feature_10', 'feature_11', 'feature_12', 'feature_13', 'feature_14', 'feature_15', 'feature_16', 'feature_17', 'feature_18', 'feature_19', 'feature_20', 'feature_21', 'feature_22', 'feature_23', 'feature_24', 'feature_25', 'feature_26', 'feature_27']
34特征重要度:
35[9, 6, 1, 15, 5, 40, 3, 0, 0, 8, 2, 1, 0, 9, 2, 0, 0, 6, 2, 6, 0, 0, 37, 2, 30, 50, 37, 29]
36加载模型用于预测
37在测试集上的rmse为:
380.4624111763226729
4.接着之前的模型继续训练
1# 继续训练
2# 从./model/model.txt中加载模型初始化
3gbm = lgb.train(params,
4 lgb_train,
5 num_boost_round=10,
6 init_model='./model/lgb_model.txt',
7 valid_sets=lgb_eval)
8
9print('以旧模型为初始化，完成第 10-20 轮训练...')
10
11# 在训练的过程中调整超参数
12# 比如这里调整的是学习率
13gbm = lgb.train(params,
14 lgb_train,
15 num_boost_round=10,
16 init_model=gbm,
17 learning_rates=lambda iter: 0.05 * (0.99 ** iter),
18 valid_sets=lgb_eval)
19
20print('逐步调整学习率完成第 20-30 轮训练...')
21
22# 调整其他超参数
23gbm = lgb.train(params,
24 lgb_train,
25 num_boost_round=10,
26 init_model=gbm,
27 valid_sets=lgb_eval,
28 callbacks=[lgb.reset_parameter(bagging_fraction=[0.7] * 5 + [0.6] * 5)])
29
30print('逐步调整bagging比率完成第 30-40 轮训练...')
31
32
33[11] valid_0's binary_logloss: 0.614214
34[12] valid_0's binary_logloss: 0.609777
35[13] valid_0's binary_logloss: 0.605236
36[14] valid_0's binary_logloss: 0.601523
37[15] valid_0's binary_logloss: 0.598256
38[16] valid_0's binary_logloss: 0.595957
39[17] valid_0's binary_logloss: 0.591773
40[18] valid_0's binary_logloss: 0.588163
41[19] valid_0's binary_logloss: 0.585106
42[20] valid_0's binary_logloss: 0.582878
43以旧模型为初始化，完成第 10-20 轮训练...
44[21] valid_0's binary_logloss: 0.614214
45[22] valid_0's binary_logloss: 0.60982
46[23] valid_0's binary_logloss: 0.605366
47[24] valid_0's binary_logloss: 0.601754
48[25] valid_0's binary_logloss: 0.598598
49[26] valid_0's binary_logloss: 0.596394
50[27] valid_0's binary_logloss: 0.59243
51[28] valid_0's binary_logloss: 0.58903
52[29] valid_0's binary_logloss: 0.586164
53[30] valid_0's binary_logloss: 0.583693
54逐步调整学习率完成第 20-30 轮训练...
55[31] valid_0's binary_logloss: 0.613881
56[32] valid_0's binary_logloss: 0.608822
57[33] valid_0's binary_logloss: 0.604746
58[34] valid_0's binary_logloss: 0.600465
59[35] valid_0's binary_logloss: 0.596407
60[36] valid_0's binary_logloss: 0.593572
61[37] valid_0's binary_logloss: 0.589196
62[38] valid_0's binary_logloss: 0.586633
63[39] valid_0's binary_logloss: 0.583136
64[40] valid_0's binary_logloss: 0.579651
65逐步调整bagging比率完成第 30-40 轮训练...
5.自定义损失函数
1# 类似在xgboost中的形式
2# 自定义损失函数需要
3def loglikelood(preds, train_data):
4 labels = train_data.get_label()
5 preds = 1. / (1. + np.exp(-preds))
6 grad = preds - labels
7 hess = preds * (1. - preds)
8 return grad, hess
9
10
11# 自定义评估函数
12def binary_error(preds, train_data):
13 labels = train_data.get_label()
14 return 'error', np.mean(labels != (preds > 0.5)), False
15
16
17gbm = lgb.train(params,
18 lgb_train,
19 num_boost_round=10,
20 init_model=gbm,
21 fobj=loglikelood,
22 feval=binary_error,
23 valid_sets=lgb_eval)
24
25print('用自定义的损失函数与评估标准完成第40-50轮...')
26
27
28[41] valid_0's binary_logloss: 4.61573 valid_0's error: 0.394
29[42] valid_0's binary_logloss: 4.66615 valid_0's error: 0.386
30[43] valid_0's binary_logloss: 4.58473 valid_0's error: 0.388
31[44] valid_0's binary_logloss: 4.63403 valid_0's error: 0.388
32[45] valid_0's binary_logloss: 4.81468 valid_0's error: 0.38
33[46] valid_0's binary_logloss: 4.86387 valid_0's error: 0.366
34[47] valid_0's binary_logloss: 4.71095 valid_0's error: 0.37
35[48] valid_0's binary_logloss: 4.81772 valid_0's error: 0.358
36[49] valid_0's binary_logloss: 4.87924 valid_0's error: 0.358
37[50] valid_0's binary_logloss: 4.86966 valid_0's error: 0.352
38用自定义的损失函数与评估标准完成第40-50轮...
sklearn与LightGBM配合使用
1.LightGBM建模，sklearn评估
1# coding: utf-8
2import lightgbm as lgb
3import pandas as pd
4from sklearn.metrics import mean_squared_error
5from sklearn.model_selection import GridSearchCV
6
7# 加载数据
8print('加载数据...')
9df_train = pd.read_csv('./data/regression.train.txt', header=None, sep='\t')
10df_test = pd.read_csv('./data/regression.test.txt', header=None, sep='\t')
11
12# 取出特征和标签
13y_train = df_train[0].values
14y_test = df_test[0].values
15X_train = df_train.drop(0, axis=1).values
16X_test = df_test.drop(0, axis=1).values
17
18print('开始训练...')
19# 直接初始化LGBMRegressor
20# 这个LightGBM的Regressor和sklearn中其他Regressor基本是一致的
21gbm = lgb.LGBMRegressor(objective='regression',
22 num_leaves=31,
23 learning_rate=0.05,
24 n_estimators=20)
25
26# 使用fit函数拟合
27gbm.fit(X_train, y_train,
28 eval_set=[(X_test, y_test)],
29 eval_metric='l1',
30 early_stopping_rounds=5)
31
32# 预测
33print('开始预测...')
34y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration_)
35# 评估预测结果
36print('预测结果的rmse是:')
37print(mean_squared_error(y_test, y_pred) ** 0.5)
38
39
40加载数据...
41开始训练...
42[1] valid_0's l1: 0.491735 valid_0's l2: 0.242763
43Training until validation scores don't improve for 5 rounds.
44[2] valid_0's l1: 0.486563 valid_0's l2: 0.237895
45[3] valid_0's l1: 0.481489 valid_0's l2: 0.233277
46[4] valid_0's l1: 0.476848 valid_0's l2: 0.22925
47[5] valid_0's l1: 0.47305 valid_0's l2: 0.226155
48[6] valid_0's l1: 0.469049 valid_0's l2: 0.222963
49[7] valid_0's l1: 0.465556 valid_0's l2: 0.220364
50[8] valid_0's l1: 0.462208 valid_0's l2: 0.217872
51[9] valid_0's l1: 0.458676 valid_0's l2: 0.215328
52[10] valid_0's l1: 0.454998 valid_0's l2: 0.212743
53[11] valid_0's l1: 0.452047 valid_0's l2: 0.210805
54[12] valid_0's l1: 0.449158 valid_0's l2: 0.208945
55[13] valid_0's l1: 0.44608 valid_0's l2: 0.206986
56[14] valid_0's l1: 0.443554 valid_0's l2: 0.205513
57[15] valid_0's l1: 0.440643 valid_0's l2: 0.203728
58[16] valid_0's l1: 0.437687 valid_0's l2: 0.201865
59[17] valid_0's l1: 0.435454 valid_0's l2: 0.200639
60[18] valid_0's l1: 0.433288 valid_0's l2: 0.199522
61[19] valid_0's l1: 0.431297 valid_0's l2: 0.198552
62[20] valid_0's l1: 0.428946 valid_0's l2: 0.197238
63Did not meet early stopping. Best iteration is:
64[20] valid_0's l1: 0.428946 valid_0's l2: 0.197238
65开始预测...
66预测结果的rmse是:
670.4441153344254208
2.网格搜索查找最优超参数
1# 配合scikit-learn的网格搜索交叉验证选择最优超参数
2estimator = lgb.LGBMRegressor(num_leaves=31)
3
4param_grid = {
5 'learning_rate': [0.01, 0.1, 1],
6 'n_estimators': [20, 40]
7}
8
9gbm = GridSearchCV(estimator, param_grid)
10
11gbm.fit(X_train, y_train)
12
13print('用网格搜索找到的最优超参数为:')
14print(gbm.best_params_)
15
16
17用网格搜索找到的最优超参数为:
18{'learning_rate': 0.1, 'n_estimators': 40}
3.绘图解释
1# coding: utf-8
2import lightgbm as lgb
3import pandas as pd
4
5try:
6 import matplotlib.pyplot as plt
7except ImportError:
8 raise ImportError('You need to install matplotlib for plotting.')
9
10# 加载数据集
11print('加载数据...')
12df_train = pd.read_csv('./data/regression.train.txt', header=None, sep='\t')
13df_test = pd.read_csv('./data/regression.test.txt', header=None, sep='\t')
14
15# 取出特征和标签
16y_train = df_train[0].values
17y_test = df_test[0].values
18X_train = df_train.drop(0, axis=1).values
19X_test = df_test.drop(0, axis=1).values
20
21# 构建lgb中的Dataset数据格式
22lgb_train = lgb.Dataset(X_train, y_train)
23lgb_test = lgb.Dataset(X_test, y_test, reference=lgb_train)
24
25# 设定参数
26params = {
27 'num_leaves': 5,
28 'metric': ('l1', 'l2'),
29 'verbose': 0
30}
31
32evals_result = {} # to record eval results for plotting
33
34print('开始训练...')
35# 训练
36gbm = lgb.train(params,
37 lgb_train,
38 num_boost_round=100,
39 valid_sets=[lgb_train, lgb_test],
40 feature_name=['f' + str(i + 1) for i in range(28)],
41 categorical_feature=[21],
42 evals_result=evals_result,
43 verbose_eval=10)
44
45print('在训练过程中绘图...')
46ax = lgb.plot_metric(evals_result, metric='l1')
47plt.show()
48
49print('画出特征重要度...')
50ax = lgb.plot_importance(gbm, max_num_features=10)
51plt.show()
52
53print('画出第84颗树...')
54ax = lgb.plot_tree(gbm, tree_index=83, figsize=(20, 8), show_info=['split_gain'])
55plt.show()
56
57#print('用graphviz画出第84颗树...')
58#graph = lgb.create_tree_digraph(gbm, tree_index=83, name='Tree84')
59#graph.render(view=True)
60
61
62加载数据...
63开始训练...
64[10] training's l1: 0.457448 training's l2: 0.217995 valid_1's l1: 0.456464 valid_1's l2: 0.21641
65[20] training's l1: 0.436869 training's l2: 0.205099 valid_1's l1: 0.434057 valid_1's l2: 0.201616
66[30] training's l1: 0.421302 training's l2: 0.197421 valid_1's l1: 0.417019 valid_1's l2: 0.192514
67[40] training's l1: 0.411107 training's l2: 0.192856 valid_1's l1: 0.406303 valid_1's l2: 0.187258
68[50] training's l1: 0.403695 training's l2: 0.189593 valid_1's l1: 0.398997 valid_1's l2: 0.183688
69[60] training's l1: 0.398704 training's l2: 0.187043 valid_1's l1: 0.393977 valid_1's l2: 0.181009
70[70] training's l1: 0.394876 training's l2: 0.184982 valid_1's l1: 0.389805 valid_1's l2: 0.178803
71[80] training's l1: 0.391147 training's l2: 0.1828 valid_1's l1: 0.386476 valid_1's l2: 0.176799
72[90] training's l1: 0.388101 training's l2: 0.180817 valid_1's l1: 0.384404 valid_1's l2: 0.175775
73[100] training's l1: 0.385174 training's l2: 0.179171 valid_1's l1: 0.382929 valid_1's l2: 0.175321
74在训练过程中绘图...
画出特征重要度...
画出第84颗树...
