experiments/different_sketch_sizes_time_bar_plots.py [138:302]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if verbose:
            print(all_residual_norms)
        residual_norms_df = pd.DataFrame.from_dict(
            all_residual_norms, orient="index"
        ).transpose()

        self.times_df = times_df
        self.residual_norms_df = residual_norms_df

        return times_df, residual_norms_df

    def run_no_momentum(self):
        run_name = "no_mom"
        algo_mode = "auto"  # no momentum
        mom_beta = None
        step_size = 1.0  # unitary step
        mom_eta = None

        no_mom_times = defaultdict(list)
        no_mom_residual_norms = {}
        counter = 1
        n_settings = 3 * len(self.solvers) * len(self.sketch_sizes)
        for solver in self.solvers:
            for i, sketch_size in enumerate(self.sketch_sizes):
                sketch_size_formula = self.sketch_size_formulas[i]
                print(
                    f"----> Setting {counter} over {n_settings} :\n"
                    f"{run_name} / {algo_mode} / {solver} / "
                    f"sketch_size = {sketch_size_formula} = {sketch_size}"
                )
                (
                    times_distribution,
                    residual_norms_distribution,
                ) = self.compute_fit_time_and_residual(
                    solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta,
                )

                # Storing the results
                self.book_keeping(
                    run_name,
                    solver,
                    sketch_size_formula,
                    sketch_size,
                    times_distribution,
                    residual_norms_distribution,
                    no_mom_times,
                    no_mom_residual_norms,
                )
                counter += 1
                print("\n")

        return no_mom_times, no_mom_residual_norms

    def run_constant_momentum(self):
        run_name = "cst_mom"  # constant momentum setting
        algo_mode = "mom"
        # REF: N Loizou paper
        mom_beta = 0.5  # constant momentum parameter
        step_size = 1.0  # constant step size TODO: justify
        mom_eta = None

        cst_mom_times = defaultdict(list)
        cst_mom_residual_norms = {}
        counter = 1 + len(self.solvers) * len(self.sketch_sizes)
        n_settings = 3 * len(self.solvers) * len(self.sketch_sizes)
        for solver in self.solvers:
            for i, sketch_size in enumerate(self.sketch_sizes):
                sketch_size_formula = self.sketch_size_formulas[i]
                print(
                    f"----> Setting {counter} over {n_settings} :\n"
                    f"{run_name} / {algo_mode} / {solver} / "
                    f"sketch_size = {sketch_size_formula} = {sketch_size}"
                )
                (
                    times_distribution,
                    residual_norms_distribution,
                ) = self.compute_fit_time_and_residual(
                    solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta,
                )

                # Storing the results
                self.book_keeping(
                    run_name,
                    solver,
                    sketch_size_formula,
                    sketch_size,
                    times_distribution,
                    residual_norms_distribution,
                    cst_mom_times,
                    cst_mom_residual_norms,
                )
                counter += 1
                print("\n")

        return cst_mom_times, cst_mom_residual_norms

    def run_increasing_momentum(self):
        run_name = "inc_mom"
        algo_mode = "mom"
        mom_beta = None
        step_size = None
        mom_eta = 0.995  # increasing momentum parameter

        inc_mom_times = defaultdict(list)
        inc_mom_residual_norms = {}
        counter = 1 + 2 * len(self.solvers) * len(self.sketch_sizes)
        n_settings = 3 * len(self.solvers) * len(self.sketch_sizes)
        for solver in self.solvers:
            for i, sketch_size in enumerate(self.sketch_sizes):
                sketch_size_formula = self.sketch_size_formulas[i]
                print(
                    f"----> Setting {counter} over {n_settings} :\n"
                    f"{run_name} / {algo_mode} / {solver} / "
                    f"sketch_size = {sketch_size_formula} = {sketch_size}"
                )
                (
                    times_distribution,
                    residual_norms_distribution,
                ) = self.compute_fit_time_and_residual(
                    solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta,
                )

                # Storing the results
                self.book_keeping(
                    run_name,
                    solver,
                    sketch_size_formula,
                    sketch_size,
                    times_distribution,
                    residual_norms_distribution,
                    inc_mom_times,
                    inc_mom_residual_norms,
                )
                counter += 1
                print("\n")

        return inc_mom_times, inc_mom_residual_norms

    def compute_fit_time_and_residual(
        self, solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta
    ):
        """
        Repeats model fit for n_repetitions.

        Returns quartile 1, 3 and median time taken
        """
        times = []
        residual_norms = []
        for repetition_idx in range(self.n_repetitions):
            print(
                f"--------> Repetition {repetition_idx+1} / " f"{self.n_repetitions}",
                end="\r",
            )

            random_state = repetition_idx
            np.random.seed(seed=random_state)
            model = load_model(
                solver,
                sketch_size,
                algo_mode,
                step_size,
                mom_beta,
                mom_eta,
                self.regularizer,
                self.is_kernel,
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



experiments/regularization_on_momentum.py [116:280]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if verbose:
            print(all_residual_norms)
        residual_norms_df = pd.DataFrame.from_dict(
            all_residual_norms, orient="index"
        ).transpose()

        self.times_df = times_df
        self.residual_norms_df = residual_norms_df

        return times_df, residual_norms_df

    def run_no_momentum(self):
        run_name = "no_mom"
        algo_mode = "auto"  # no momentum
        mom_beta = None
        step_size = 1.0  # unitary step
        mom_eta = None

        no_mom_times = defaultdict(list)
        no_mom_residual_norms = {}
        counter = 1
        n_settings = 3 * len(self.solvers) * len(self.sketch_sizes)
        for solver in self.solvers:
            for i, sketch_size in enumerate(self.sketch_sizes):
                sketch_size_formula = self.sketch_size_formulas[i]
                print(
                    f"----> Setting {counter} over {n_settings} :\n"
                    f"{run_name} / {algo_mode} / {solver} / "
                    f"sketch_size = {sketch_size_formula} = {sketch_size}"
                )
                (
                    times_distribution,
                    residual_norms_distribution,
                ) = self.compute_fit_time_and_residual(
                    solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta,
                )

                # Storing the results
                self.book_keeping(
                    run_name,
                    solver,
                    sketch_size_formula,
                    sketch_size,
                    times_distribution,
                    residual_norms_distribution,
                    no_mom_times,
                    no_mom_residual_norms,
                )
                counter += 1
                print("\n")

        return no_mom_times, no_mom_residual_norms

    def run_constant_momentum(self):
        run_name = "cst_mom"  # constant momentum setting
        algo_mode = "mom"
        # REF: N Loizou paper
        mom_beta = 0.5  # constant momentum parameter
        step_size = 1.0  # constant step size TODO: justify
        mom_eta = None

        cst_mom_times = defaultdict(list)
        cst_mom_residual_norms = {}
        counter = 1 + len(self.solvers) * len(self.sketch_sizes)
        n_settings = 3 * len(self.solvers) * len(self.sketch_sizes)
        for solver in self.solvers:
            for i, sketch_size in enumerate(self.sketch_sizes):
                sketch_size_formula = self.sketch_size_formulas[i]
                print(
                    f"----> Setting {counter} over {n_settings} :\n"
                    f"{run_name} / {algo_mode} / {solver} / "
                    f"sketch_size = {sketch_size_formula} = {sketch_size}"
                )
                (
                    times_distribution,
                    residual_norms_distribution,
                ) = self.compute_fit_time_and_residual(
                    solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta,
                )

                # Storing the results
                self.book_keeping(
                    run_name,
                    solver,
                    sketch_size_formula,
                    sketch_size,
                    times_distribution,
                    residual_norms_distribution,
                    cst_mom_times,
                    cst_mom_residual_norms,
                )
                counter += 1
                print("\n")

        return cst_mom_times, cst_mom_residual_norms

    def run_increasing_momentum(self):
        run_name = "inc_mom"
        algo_mode = "mom"
        mom_beta = None
        step_size = None
        mom_eta = 0.995  # increasing momentum parameter

        inc_mom_times = defaultdict(list)
        inc_mom_residual_norms = {}
        counter = 1 + 2 * len(self.solvers) * len(self.sketch_sizes)
        n_settings = 3 * len(self.solvers) * len(self.sketch_sizes)
        for solver in self.solvers:
            for i, sketch_size in enumerate(self.sketch_sizes):
                sketch_size_formula = self.sketch_size_formulas[i]
                print(
                    f"----> Setting {counter} over {n_settings} :\n"
                    f"{run_name} / {algo_mode} / {solver} / "
                    f"sketch_size = {sketch_size_formula} = {sketch_size}"
                )
                (
                    times_distribution,
                    residual_norms_distribution,
                ) = self.compute_fit_time_and_residual(
                    solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta,
                )

                # Storing the results
                self.book_keeping(
                    run_name,
                    solver,
                    sketch_size_formula,
                    sketch_size,
                    times_distribution,
                    residual_norms_distribution,
                    inc_mom_times,
                    inc_mom_residual_norms,
                )
                counter += 1
                print("\n")

        return inc_mom_times, inc_mom_residual_norms

    def compute_fit_time_and_residual(
        self, solver, sketch_size, algo_mode, step_size, mom_beta, mom_eta
    ):
        """
        Repeats model fit for n_repetitions.

        Returns quartile 1, 3 and median time taken
        """
        times = []
        residual_norms = []
        for repetition_idx in range(self.n_repetitions):
            print(
                f"--------> Repetition {repetition_idx+1} / " f"{self.n_repetitions}",
                end="\r",
            )

            random_state = repetition_idx
            np.random.seed(seed=random_state)
            model = load_model(
                solver,
                sketch_size,
                algo_mode,
                step_size,
                mom_beta,
                mom_eta,
                self.regularizer,
                self.is_kernel,
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -