experiments/different_sketch_sizes_time_bar_plots.py [138:400]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if verbose: print(all_residual_norms) residual_norms_df = pd.DataFrame.from_dict( all_residual_norms, orient="index" ).transpose() self.times_df = times_df self.residual_norms_df = residual_norms_df return times_df, residual_norms_df def run_no_momentum(self): run_name = "no_mom" algo_mode = "auto" # no momentum mom_beta = None step_size = 1.0 # unitary step mom_eta = None no_mom_times = defaultdict(list) no_mom_residual_norms = {} counter = 1 n_settings = 3 * len(self.solvers) * len(self.sketch_sizes) for solver in self.solvers: for i, sketch_size in enumerate(self.sketch_sizes): sketch_size_formula = self.sketch_size_formulas[i] print( f"----> Setting {counter} over {n_settings} :\n" f"{run_name} / {algo_mode} / {solver} / " f"sketch_size = {sketch_size_formula} = {sketch_size}" ) ( times_distribution, residual_norms_distribution, ) = self.compute_fit_time_and_residual( solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta, ) # Storing the results self.book_keeping( run_name, solver, sketch_size_formula, sketch_size, times_distribution, residual_norms_distribution, no_mom_times, no_mom_residual_norms, ) counter += 1 print("\n") return no_mom_times, no_mom_residual_norms def run_constant_momentum(self): run_name = "cst_mom" # constant momentum setting algo_mode = "mom" # REF: N Loizou paper mom_beta = 0.5 # constant momentum parameter step_size = 1.0 # constant step size TODO: justify mom_eta = None cst_mom_times = defaultdict(list) cst_mom_residual_norms = {} counter = 1 + len(self.solvers) * len(self.sketch_sizes) n_settings = 3 * len(self.solvers) * len(self.sketch_sizes) for solver in self.solvers: for i, sketch_size in enumerate(self.sketch_sizes): sketch_size_formula = self.sketch_size_formulas[i] print( f"----> Setting {counter} over {n_settings} :\n" f"{run_name} / {algo_mode} / {solver} / " f"sketch_size = {sketch_size_formula} = {sketch_size}" ) ( times_distribution, residual_norms_distribution, ) = self.compute_fit_time_and_residual( solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta, ) # Storing the results self.book_keeping( run_name, solver, sketch_size_formula, sketch_size, times_distribution, residual_norms_distribution, cst_mom_times, cst_mom_residual_norms, ) counter += 1 print("\n") return cst_mom_times, cst_mom_residual_norms def run_increasing_momentum(self): run_name = "inc_mom" algo_mode = "mom" mom_beta = None step_size = None mom_eta = 0.995 # increasing momentum parameter inc_mom_times = defaultdict(list) inc_mom_residual_norms = {} counter = 1 + 2 * len(self.solvers) * len(self.sketch_sizes) n_settings = 3 * len(self.solvers) * len(self.sketch_sizes) for solver in self.solvers: for i, sketch_size in enumerate(self.sketch_sizes): sketch_size_formula = self.sketch_size_formulas[i] print( f"----> Setting {counter} over {n_settings} :\n" f"{run_name} / {algo_mode} / {solver} / " f"sketch_size = {sketch_size_formula} = {sketch_size}" ) ( times_distribution, residual_norms_distribution, ) = self.compute_fit_time_and_residual( solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta, ) # Storing the results self.book_keeping( run_name, solver, sketch_size_formula, sketch_size, times_distribution, residual_norms_distribution, inc_mom_times, inc_mom_residual_norms, ) counter += 1 print("\n") return inc_mom_times, inc_mom_residual_norms def compute_fit_time_and_residual( self, solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta ): """ Repeats model fit for n_repetitions. Returns quartile 1, 3 and median time taken """ times = [] residual_norms = [] for repetition_idx in range(self.n_repetitions): print( f"--------> Repetition {repetition_idx+1} / " f"{self.n_repetitions}", end="\r", ) random_state = repetition_idx np.random.seed(seed=random_state) model = load_model( solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta, self.regularizer, self.is_kernel, self.use_heuristic, self.tolerance, self.max_iter, random_state, ) start = default_timer() model.fit(self.X, self.y) times.append(default_timer() - start) residual_norms.append(model.residual_norms) times_distribution = compute_distribution(times) residual_norms = pad_residual_norms(residual_norms) residual_norms_distribution = compute_distribution(residual_norms) return times_distribution, residual_norms_distribution def book_keeping( self, run_name, solver, sketch_size_formula, sketch_size, times_distribution, residual_norms_distribution, run_times, run_residual_norms, ): """ Store time and relative residual norms distribution for each parametrization. run_name : - "cst_mom" for constant momentum - "inc_mom" for increasing momentum - "no_mom" for no momentum """ description = { "run_name": run_name, "solver": solver, "sketch_size_formula": sketch_size_formula, "sketch_size": sketch_size, } description_string = ( f"{run_name} | {solver} | sketch_size = " f"{sketch_size_formula} = {sketch_size}" ) run_time = { **description, "time (median)": times_distribution.median, "time (1st quartile)": times_distribution.q1, "time (3rd quartile)": times_distribution.q3, } run_times = update_times(run_times, run_time) run_residual_norm = { f"{description_string} | residual_norms (median)": residual_norms_distribution.median, f"{description_string} | residual_norms (1st quartile)": residual_norms_distribution.q1, f"{description_string} | residual_norms (3rd quartile)": residual_norms_distribution.q3, } run_residual_norms.update(run_residual_norm) def save(self, exp_name, save_path): """Saves benchmark results, plots and configs""" full_path = os.path.join(save_path, exp_name) # clear old results shutil.rmtree(full_path, ignore_errors=True) os.makedirs(full_path) self.save_settings(exp_name, full_path) self.save_results(exp_name, full_path) self.save_plots(exp_name, full_path) def save_settings(self, exp_name, save_path): with open( os.path.join(save_path, exp_name + "_settings.txt"), "w", ) as text_file: settings_dict = { "dataset_name": self.dataset_name, "regularizer": self.regularizer, "tolerance": self.tolerance, "is_kernel": self.is_kernel, "n_repetitions": self.n_repetitions, "solvers": self.solvers, "sketch_sizes": self.sketch_sizes, "sketch_size_formulas": self.sketch_size_formulas, } print(f"Settings: {settings_dict}", file=text_file) def save_results(self, exp_name, save_path): self.times_df.to_csv(os.path.join(save_path, exp_name + "_times.csv")) self.residual_norms_df.to_csv( os.path.join(save_path, exp_name + "_residual_norms.csv") ) def save_plots(self, exp_name, save_path): for i, sketch_size in enumerate(self.sketch_sizes): sketch_size_formula = self.sketch_size_formulas[i] for solver in self.solvers: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - experiments/momentum_regimes.py [133:395]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if verbose: print(all_residual_norms) residual_norms_df = pd.DataFrame.from_dict( all_residual_norms, orient="index" ).transpose() self.times_df = times_df self.residual_norms_df = residual_norms_df return times_df, residual_norms_df def run_no_momentum(self): run_name = "no_mom" algo_mode = "auto" # no momentum mom_beta = None step_size = 1.0 # unitary step mom_eta = None no_mom_times = defaultdict(list) no_mom_residual_norms = {} counter = 1 n_settings = 3 * len(self.solvers) * len(self.sketch_sizes) for solver in self.solvers: for i, sketch_size in enumerate(self.sketch_sizes): sketch_size_formula = self.sketch_size_formulas[i] print( f"----> Setting {counter} over {n_settings} :\n" f"{run_name} / {algo_mode} / {solver} / " f"sketch_size = {sketch_size_formula} = {sketch_size}" ) ( times_distribution, residual_norms_distribution, ) = self.compute_fit_time_and_residual( solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta, ) # Storing the results self.book_keeping( run_name, solver, sketch_size_formula, sketch_size, times_distribution, residual_norms_distribution, no_mom_times, no_mom_residual_norms, ) counter += 1 print("\n") return no_mom_times, no_mom_residual_norms def run_constant_momentum(self): run_name = "cst_mom" # constant momentum setting algo_mode = "mom" # REF: N Loizou paper mom_beta = 0.5 # constant momentum parameter step_size = 1.0 # constant step size TODO: justify mom_eta = None cst_mom_times = defaultdict(list) cst_mom_residual_norms = {} counter = 1 + len(self.solvers) * len(self.sketch_sizes) n_settings = 3 * len(self.solvers) * len(self.sketch_sizes) for solver in self.solvers: for i, sketch_size in enumerate(self.sketch_sizes): sketch_size_formula = self.sketch_size_formulas[i] print( f"----> Setting {counter} over {n_settings} :\n" f"{run_name} / {algo_mode} / {solver} / " f"sketch_size = {sketch_size_formula} = {sketch_size}" ) ( times_distribution, residual_norms_distribution, ) = self.compute_fit_time_and_residual( solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta, ) # Storing the results self.book_keeping( run_name, solver, sketch_size_formula, sketch_size, times_distribution, residual_norms_distribution, cst_mom_times, cst_mom_residual_norms, ) counter += 1 print("\n") return cst_mom_times, cst_mom_residual_norms def run_increasing_momentum(self): run_name = "inc_mom" algo_mode = "mom" mom_beta = None step_size = None mom_eta = 0.995 # increasing momentum parameter inc_mom_times = defaultdict(list) inc_mom_residual_norms = {} counter = 1 + 2 * len(self.solvers) * len(self.sketch_sizes) n_settings = 3 * len(self.solvers) * len(self.sketch_sizes) for solver in self.solvers: for i, sketch_size in enumerate(self.sketch_sizes): sketch_size_formula = self.sketch_size_formulas[i] print( f"----> Setting {counter} over {n_settings} :\n" f"{run_name} / {algo_mode} / {solver} / " f"sketch_size = {sketch_size_formula} = {sketch_size}" ) ( times_distribution, residual_norms_distribution, ) = self.compute_fit_time_and_residual( solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta, ) # Storing the results self.book_keeping( run_name, solver, sketch_size_formula, sketch_size, times_distribution, residual_norms_distribution, inc_mom_times, inc_mom_residual_norms, ) counter += 1 print("\n") return inc_mom_times, inc_mom_residual_norms def compute_fit_time_and_residual( self, solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta ): """ Repeats model fit for n_repetitions. Returns quartile 1, 3 and median time taken """ times = [] residual_norms = [] for repetition_idx in range(self.n_repetitions): print( f"--------> Repetition {repetition_idx+1} / " f"{self.n_repetitions}", end="\r", ) random_state = repetition_idx np.random.seed(seed=random_state) model = load_model( solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta, self.regularizer, self.is_kernel, self.use_heuristic, self.tolerance, self.max_iter, random_state, ) start = default_timer() model.fit(self.X, self.y) times.append(default_timer() - start) residual_norms.append(model.residual_norms) times_distribution = compute_distribution(times) residual_norms = pad_residual_norms(residual_norms) residual_norms_distribution = compute_distribution(residual_norms) return times_distribution, residual_norms_distribution def book_keeping( self, run_name, solver, sketch_size_formula, sketch_size, times_distribution, residual_norms_distribution, run_times, run_residual_norms, ): """ Store time and relative residual norms distribution for each parametrization. run_name : - "cst_mom" for constant momentum - "inc_mom" for increasing momentum - "no_mom" for no momentum """ description = { "run_name": run_name, "solver": solver, "sketch_size_formula": sketch_size_formula, "sketch_size": sketch_size, } description_string = ( f"{run_name} | {solver} | sketch_size = " f"{sketch_size_formula} = {sketch_size}" ) run_time = { **description, "time (median)": times_distribution.median, "time (1st quartile)": times_distribution.q1, "time (3rd quartile)": times_distribution.q3, } run_times = update_times(run_times, run_time) run_residual_norm = { f"{description_string} | residual_norms (median)": residual_norms_distribution.median, f"{description_string} | residual_norms (1st quartile)": residual_norms_distribution.q1, f"{description_string} | residual_norms (3rd quartile)": residual_norms_distribution.q3, } run_residual_norms.update(run_residual_norm) def save(self, exp_name, save_path): """Saves benchmark results, plots and configs""" full_path = os.path.join(save_path, exp_name) # clear old results shutil.rmtree(full_path, ignore_errors=True) os.makedirs(full_path) self.save_settings(exp_name, full_path) self.save_results(exp_name, full_path) self.save_plots(exp_name, full_path) def save_settings(self, exp_name, save_path): with open( os.path.join(save_path, exp_name + "_settings.txt"), "w", ) as text_file: settings_dict = { "dataset_name": self.dataset_name, "regularizer": self.regularizer, "tolerance": self.tolerance, "is_kernel": self.is_kernel, "n_repetitions": self.n_repetitions, "solvers": self.solvers, "sketch_sizes": self.sketch_sizes, "sketch_size_formulas": self.sketch_size_formulas, } print(f"Settings: {settings_dict}", file=text_file) def save_results(self, exp_name, save_path): self.times_df.to_csv(os.path.join(save_path, exp_name + "_times.csv")) self.residual_norms_df.to_csv( os.path.join(save_path, exp_name + "_residual_norms.csv") ) def save_plots(self, exp_name, save_path): for i, sketch_size in enumerate(self.sketch_sizes): sketch_size_formula = self.sketch_size_formulas[i] for solver in self.solvers: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -