in tabular/src/autogluon/tabular/models/lgb/lgb_model.py [0:0]
def _fit(self,
X,
y,
X_val=None,
y_val=None,
time_limit=None,
num_gpus=0,
num_cpus=0,
sample_weight=None,
sample_weight_val=None,
verbosity=2,
**kwargs):
try_import_lightgbm() # raise helpful error message if LightGBM isn't installed
start_time = time.time()
ag_params = self._get_ag_params()
params = self._get_model_params()
params['num_threads'] = num_cpus
params = fixedvals_from_searchspaces(params)
if verbosity <= 1:
log_period = False
elif verbosity == 2:
log_period = 1000
elif verbosity == 3:
log_period = 50
else:
log_period = 1
stopping_metric, stopping_metric_name = self._get_stopping_metric_internal()
if self.problem_type in [MULTICLASS, SOFTCLASS] and 'num_classes' not in params:
params['num_classes'] = self.num_classes
num_boost_round = params.pop('num_boost_round', 1000)
dart_retrain = params.pop('dart_retrain', False) # Whether to retrain the model to get optimal iteration if model is trained in 'dart' mode.
if num_gpus != 0:
if 'device' not in params:
# TODO: lightgbm must have a special install to support GPU: https://github.com/Microsoft/LightGBM/tree/master/python-package#build-gpu-version
# Before enabling GPU, we should add code to detect that GPU-enabled version is installed and that a valid GPU exists.
# GPU training heavily alters accuracy, often in a negative manner. We will have to be careful about when to use GPU.
params['device'] = 'gpu'
logger.log(20, f'\tTraining {self.name} with GPU, note that this may negatively impact model quality compared to CPU training.')
logger.log(15, f'Training Gradient Boosting Model for {num_boost_round} rounds...')
logger.log(15, "with the following hyperparameter settings:")
logger.log(15, params)
num_rows_train = len(X)
dataset_train, dataset_val = self.generate_datasets(
X=X, y=y, params=params, X_val=X_val, y_val=y_val,
sample_weight=sample_weight, sample_weight_val=sample_weight_val
)
gc.collect()
callbacks = []
valid_names = []
valid_sets = []
if dataset_val is not None:
from .callbacks import early_stopping_custom
# TODO: Better solution: Track trend to early stop when score is far worse than best score, or score is trending worse over time
early_stopping_rounds = ag_params.get('ag.early_stop', 'adaptive')
if isinstance(early_stopping_rounds, (str, tuple, list)):
early_stopping_rounds = self._get_early_stopping_rounds(num_rows_train=num_rows_train, strategy=early_stopping_rounds)
if early_stopping_rounds is None:
early_stopping_rounds = 999999
reporter = kwargs.get('reporter', None)
train_loss_name = self._get_train_loss_name() if reporter is not None else None
if train_loss_name is not None:
if 'metric' not in params or params['metric'] == '':
params['metric'] = train_loss_name
elif train_loss_name not in params['metric']:
params['metric'] = f'{params["metric"]},{train_loss_name}'
callbacks += [
# Note: Don't use self.params_aux['max_memory_usage_ratio'] here as LightGBM handles memory per iteration optimally. # TODO: Consider using when ratio < 1.
early_stopping_custom(early_stopping_rounds, metrics_to_use=[('valid_set', stopping_metric_name)], max_diff=None, start_time=start_time, time_limit=time_limit,
ignore_dart_warning=True, verbose=False, manual_stop_file=False, reporter=reporter, train_loss_name=train_loss_name),
]
valid_names = ['valid_set'] + valid_names
valid_sets = [dataset_val] + valid_sets
from lightgbm.callback import log_evaluation
if log_period is not None:
callbacks.append(log_evaluation(period=log_period))
seed_val = params.pop('seed_value', 0)
train_params = {
'params': params,
'train_set': dataset_train,
'num_boost_round': num_boost_round,
'valid_sets': valid_sets,
'valid_names': valid_names,
'callbacks': callbacks,
}
if not isinstance(stopping_metric, str):
train_params['feval'] = stopping_metric
else:
if 'metric' not in train_params['params'] or train_params['params']['metric'] == '':
train_params['params']['metric'] = stopping_metric
elif stopping_metric not in train_params['params']['metric']:
train_params['params']['metric'] = f'{train_params["params"]["metric"]},{stopping_metric}'
if self.problem_type == SOFTCLASS:
train_params['fobj'] = lgb_utils.softclass_lgbobj
if seed_val is not None:
train_params['params']['seed'] = seed_val
random.seed(seed_val)
np.random.seed(seed_val)
# Train LightGBM model:
import lightgbm as lgb
from lightgbm.basic import LightGBMError
with warnings.catch_warnings():
# Filter harmless warnings introduced in lightgbm 3.0, future versions plan to remove: https://github.com/microsoft/LightGBM/issues/3379
warnings.filterwarnings('ignore', message='Overriding the parameters from Reference Dataset.')
warnings.filterwarnings('ignore', message='categorical_column in param dict is overridden.')
try:
self.model = lgb.train(**train_params)
except LightGBMError:
if train_params['params'].get('device', 'cpu') != 'gpu':
raise
else:
logger.warning('Warning: GPU mode might not be installed for LightGBM, GPU training raised an exception. Falling back to CPU training...'
'Refer to LightGBM GPU documentation: https://github.com/Microsoft/LightGBM/tree/master/python-package#build-gpu-version'
'One possible method is:'
'\tpip uninstall lightgbm -y'
'\tpip install lightgbm --install-option=--gpu'
)
train_params['params']['device'] = 'cpu'
self.model = lgb.train(**train_params)
retrain = False
if train_params['params'].get('boosting_type', '') == 'dart':
if dataset_val is not None and dart_retrain and (self.model.best_iteration != num_boost_round):
retrain = True
if time_limit is not None:
time_left = time_limit + start_time - time.time()
if time_left < 0.5 * time_limit:
retrain = False
if retrain:
logger.log(15, f"Retraining LGB model to optimal iterations ('dart' mode).")
train_params.pop('callbacks')
train_params['num_boost_round'] = self.model.best_iteration
self.model = lgb.train(**train_params)
else:
logger.log(15, f"Not enough time to retrain LGB model ('dart' mode)...")
if dataset_val is not None and not retrain:
self.params_trained['num_boost_round'] = self.model.best_iteration
else:
self.params_trained['num_boost_round'] = self.model.current_iteration()