files/gitlab-ctl-commands-ee/lib/geo/replication.rb

require 'io/console'
require 'rainbow/ext/string'

# For testing purposes, if the first path cannot be found load the second
begin
  require_relative '../../../omnibus-ctl/lib/postgresql'
rescue LoadError
  require_relative '../../../gitlab-ctl-commands/lib/postgresql'
end

module Geo
  class Replication
    attr_accessor :base_path, :data_path, :postgresql_dir_path, :tmp_dir, :ctl
    attr_writer :data_dir, :tmp_data_dir
    attr_reader :options

    DEFAULT_REPLICATION_TIMEOUT_S = 12 * 60 * 60 # 12 hours

    def initialize(instance, options)
      @base_path = instance.base_path
      @data_path = instance.data_path
      @postgresql_dir_path = GitlabCtl::Util.get_public_node_attributes.dig('postgresql', 'dir')
      @ctl = instance
      @options = options
    end

    def postgresql_user
      @postgresql_user ||= GitlabCtl::PostgreSQL.postgresql_username
    end

    def postgresql_group
      @postgresql_group ||= GitlabCtl::PostgreSQL.postgresql_group
    end

    def postgresql_version
      @postgresql_version ||= GitlabCtl::PostgreSQL.postgresql_version
    end

    def check_gitlab_active?
      return unless gitlab_is_active?

      if @options[:force]
        puts "Found data inside the #{db_name} database! Proceeding because --force was supplied".color(:yellow)
      else
        puts "Found data inside the #{db_name} database! If you are sure you are in the secondary server, override with --force".color(:red)
        exit 1
      end
    end

    def check_service_enabled?
      return if ctl.service_enabled?('postgresql')

      puts 'There is no PostgreSQL instance enabled in omnibus, exiting...'.color(:red)
      Kernel.exit 1
    end

    def confirm_replication
      return if @options[:now]

      puts '*** Are you sure you want to continue (replicate/no)? ***'.color(:yellow)

      loop do
        print 'Confirmation: '
        answer = STDIN.gets.to_s.strip

        break if answer == 'replicate'

        exit 0 if answer == 'no'

        puts "*** You entered `#{answer}` instead of `replicate` or `no`.".color(:red)
      end
    end

    def print_warning
      puts
      puts '---------------------------------------------------------------'.color(:yellow)
      puts 'WARNING: Make sure this script is run from the secondary server'.color(:yellow)
      puts '---------------------------------------------------------------'.color(:yellow)
      puts
      puts '*** You are about to delete your local PostgreSQL database, and replicate the primary database. ***'.color(:yellow)
      puts "*** The primary geo node is `#{@options[:host]}` ***".color(:yellow)
      puts
    end

    def execute
      check_gitlab_active?
      check_service_enabled?

      print_warning
      confirm_replication

      @options[:password] = ask_pass

      create_gitlab_backup!

      puts '* Stopping PostgreSQL and all GitLab services'.color(:green)
      run_command('gitlab-ctl stop')

      @pgpass = "#{postgresql_dir_path}/.pgpass"
      create_pgpass_file!

      check_and_create_replication_slot!

      orig_conf = "#{postgresql_dir_path}/data/postgresql.conf"
      if File.exist?(orig_conf)
        puts '* Backing up postgresql.conf'.color(:green)
        run_command("mv #{orig_conf} #{postgresql_dir_path}/")
      end

      bkp_dir = "#{postgresql_dir_path}/data.#{Time.now.to_i}"
      puts "* Moving old data directory to '#{bkp_dir}'".color(:green)

      run_command("mv #{postgresql_dir_path}/data #{bkp_dir}")

      puts "* Starting base backup as the replicator user (#{@options[:user]})".color(:green)

      run_command(pg_basebackup_command,
                  live: true, timeout: backup_timeout)

      puts '* Restoring postgresql.conf'.color(:green)
      run_command("mv #{postgresql_dir_path}/postgresql.conf #{postgresql_dir_path}/data/")

      write_replication_settings!

      puts '* Setting ownership permissions in PostgreSQL data directory'.color(:green)
      run_command("chown -R #{postgresql_user}:#{postgresql_group} #{postgresql_dir_path}/data")

      puts '* Starting PostgreSQL and all GitLab services'.color(:green)
      run_command('gitlab-ctl start')
    end

    def check_and_create_replication_slot!
      return if @options[:skip_replication_slot]

      puts "* Checking for replication slot #{@options[:slot_name]}".color(:green)
      return if replication_slot_exists?

      puts "* Creating replication slot #{@options[:slot_name]}".color(:green)
      create_replication_slot!
    end

    def write_replication_settings!
      write_recovery_settings!
      create_standby_file!
    end

    private

    def backup_timeout
      @options[:backup_timeout] || DEFAULT_REPLICATION_TIMEOUT_S
    end

    def create_gitlab_backup!
      return if @options[:skip_backup]
      return unless gitlab_bootstrapped? && database_exists? && table_exists?('projects')

      puts '* Executing GitLab backup task to prevent accidental data loss'.color(:green)
      run_command('gitlab-rake gitlab:backup:create')
    end

    def create_pgpass_file!
      File.open(@pgpass, 'w', 0600) do |file|
        file.write(<<~EOF
          #{@options[:host]}:#{@options[:port]}:*:#{@options[:user]}:#{@options[:password]}
        EOF
                  )
      end
      run_command("chown #{postgresql_user}:#{postgresql_group} #{@pgpass}")
    end

    def write_recovery_settings!
      geo_conf_file = "#{postgresql_dir_path}/data/gitlab-geo.conf"
      File.open(geo_conf_file, "w", 0640) do |file|
        settings = <<~EOF
          # - Added by GitLab Omnibus for Geo replication -
          recovery_target_timeline = '#{@options[:recovery_target_timeline]}'
          primary_conninfo = 'host=#{@options[:host]} port=#{@options[:port]} user=#{@options[:user]} password=#{@options[:password]} sslmode=#{@options[:sslmode]} sslcompression=#{@options[:sslcompression]}'
        EOF

        file.write(settings)
        file.write("primary_slot_name = '#{@options[:slot_name]}'\n") if @options[:slot_name]
      end
    end

    def create_standby_file!
      standby_file = "#{postgresql_dir_path}/data/standby.signal"
      File.write(standby_file, "")
      run_command("chown #{postgresql_user}:#{postgresql_group} #{standby_file}")
    end

    def ask_pass
      GitlabCtl::Util.get_password(input_text: "Enter the password for #{@options[:user]}@#{@options[:host]}: ", do_confirm: false)
    end

    def replication_slot_exists?
      status = run_psql_command("SELECT slot_name FROM pg_replication_slots WHERE slot_name = '#{@options[:slot_name]}';")
      status.stdout.include?(@options[:slot_name])
    end

    def create_replication_slot!
      status = run_psql_command("SELECT slot_name FROM pg_create_physical_replication_slot('#{@options[:slot_name]}');")
      status.stdout.include?(@options[:slot_name])
    end

    def pg_basebackup_command
      slot_arguments =
        if @options[:skip_replication_slot]
          ''
        else
          "-S #{@options[:slot_name]}"
        end

      %W(
        PGPASSFILE=#{@pgpass} #{@base_path}/embedded/bin/pg_basebackup
        -h #{@options[:host]}
        -p #{@options[:port]}
        -D #{@postgresql_dir_path}/data
        -U #{@options[:user]}
        -v
        -P
        -X stream
        #{slot_arguments}
      ).join(' ')
    end

    def run_psql_command(query)
      cmd = %(PGPASSFILE=#{@pgpass} #{base_path}/bin/gitlab-psql -h #{@options[:host]} -p #{@options[:port]} -U #{@options[:user]} -d #{db_name} -t -c "#{query}")
      run_command(cmd, live: false)
    end

    def run_command(cmd, live: false, timeout: nil)
      status = GitlabCtl::Util.run_command(cmd, live: live, timeout: timeout)
      if status.error?
        puts status.stdout
        puts status.stderr
        teardown(cmd)
      end

      status
    end

    def run_query(query)
      status = GitlabCtl::Util.run_command(
        "#{base_path}/bin/gitlab-psql -d #{db_name} -c '#{query}' -q -t"
      )
      status.error? ? false : status.stdout.strip
    end

    def gitlab_bootstrapped?
      File.exist?("#{data_path}/bootstrapped")
    end

    def database_exists?
      status = GitlabCtl::Util.run_command("#{base_path}/bin/gitlab-psql -d template1 -c 'SELECT datname FROM pg_database' -A | grep -x #{db_name}")
      !status.error?
    end

    def table_exists?(table_name)
      query = "SELECT table_name
                 FROM information_schema.tables
                WHERE table_catalog = '#{db_name}'
                  AND table_schema='public'"
      status = GitlabCtl::Util.run_command("#{base_path}/bin/gitlab-psql -d #{db_name} -c \"#{query}\" -A | grep -x #{table_name}")
      !status.error?
    end

    def table_empty?(table_name)
      output = run_query('SELECT 1 FROM projects LIMIT 1')
      output == '1' ? false : true
    end

    def gitlab_is_active?
      system("gitlab-rake", "gitlab:db:active")
    end

    def db_name
      @options[:db_name]
    end

    def teardown(cmd)
      puts <<~MESSAGE.color(:red)
        *** Initial replication failed! ***

        Replication tool returned with a non zero exit status!

        Troubleshooting tips:
          - replication should be run by root user
          - check if `roles ['geo_primary_role']` or `geo_primary_role['enable'] = true` exists in `gitlab.rb` on the primary node
          - check your trust settings `md5_auth_cidr_addresses` in `gitlab.rb` on the primary node

        Failed to execute: #{cmd}
      MESSAGE

      exit 1
    end
  end
end