tools/merge_subscriptions.rb

#!/usr/bin/env ruby

# @(#) merge subs information from old and new mail list servers during changeover

#        ***********  DRAFT  *****************
#        ***********  DRAFT  *****************

# The mail list server is being migrated to a new host.
# This means some mail lists are on a new host whilst some are on the old host.
# It looks as though the old list files are not being removed, so merging must
# use entries from the new host, filling in from the old host where necessary.

# To allow the mail parsing utils to continue to work as far as possible during
# the changeover, we merge the old and new data files.

# Assume we have new data under /NEW/;
# old data under /OLD/; and
# output is under /OUT/.

# For each entry under /NEW/cache, create a link under /OUT/cache/, replacing any existing entry.
# For each file under /OLD/cache, create a link under /OUT/cache/ -- if none exists

# The following top-level files also need to be merged:
# qmail.ids = read both files, and eliminate duplicates
# list-flags = read old file, storing in hash (dom+list); merge in new; write out merged
# list-start - copy from NEW directory at end of run
# list-counts - merge (apart from total)

require 'find'
require 'fileutils'

def merge_files(old_host, new_host, out)
  # merge list-counts
  counts = {}
  File.open(File.join(old_host, 'list-counts')).each do |line|
    if line =~ %r{^( *\d+) (\S+)}
      counts[$2] = $1
    else
      puts "Unexpected count line #{line}"
    end
  end

  File.open(File.join(new_host, 'list-counts')).each do |line|
    if line =~ %r{^( *\d+) (\S+)}
      counts[$2] = $1
    else
      puts "Unexpected count line #{line}"
    end
  end

  File.open(File.join(out, 'list-counts'), 'w') do |f|
    counts.delete('total') # does not sort correctly; and cannot be merged
    counts.sort.each do |k, v|
      f.puts "#{v} #{k}"
    end
  end

  # merge list-flags
  old_flags = File.join(old_host, 'list-flags')
  new_flags = File.join(new_host, 'list-flags')
  out_flags = File.join(out, 'list-flags')
  out_time = File.mtime(out_flags) rescue 0
  # only update flags if they have changed
  if File.mtime(old_flags) > out_time || File.mtime(new_flags) > out_time

    flags = {}
    File.open(old_flags).each do |line|
      parts = line.chomp.split(' ', 2)
      flags[parts[1]] = parts[0]
    end

    File.open(new_flags).each do |line|
      parts = line.chomp.split(' ', 2)
      flags[parts[1]] = parts[0]
    end

    File.open(out_flags, 'w') do |f|
      flags.sort.each do |k, v|
        f.puts "#{v} #{k}"
      end
    end

  end

    # Create links to new cache files as necessary
  Find.find(File.join(new_host, 'cache')) do |path|
    if File.file? path
      targ = path.sub(new_host, out)
      dir = File.dirname(targ)
      unless File.directory? dir
        # puts "Making #{dir}"
        FileUtils.mkdir_p dir
      end
      # puts "Linking #{targ}"
      # Always create a link
      FileUtils.symlink path, targ, force: true
    end
  end

  # Now fill in any gaps from the old host
  Find.find(File.join(old_host, 'cache')) do |path|
    if File.file? path
      targ = path.sub(old_host, out)
      # At the start of a merge, there may be plain files from before
      # These need to be replaced with soft links
      unless File.exist?(targ) && File.ftype(targ) == 'link'
        dir = File.dirname(targ)
        unless File.directory? dir
          # puts "Making #{dir}"
          FileUtils.mkdir_p dir
        end
        # puts "Linking #{targ}"
        FileUtils.symlink path, targ, force: true
      end
    end
  end

  # Remove any dangling files/links (e.g. after a list is removed)
  Find.find(File.join(out, 'cache')) do |path|
    next if File.directory? path
    unless File.exist?(path) && File.ftype(path) == 'link'
      $stderr.puts "WARN: Removing real file or missing link #{path}"
      begin
        File.unlink(path)
      rescue StandardError => e
        $stderr.puts "WARN: Failed to remove path #{e.inspect}"
      end
    end
  end

  # Update the timestamp
  FileUtils.copy_file(File.join(new_host, 'list-start'), File.join(out, 'list-start'), true)
end

# merge qmail.ids
def merge_qmail(old_host, new_host, out)
  merged = File.read(File.join(old_host, 'qmail.ids')).
    split("\n").union(File.read(File.join(new_host, 'qmail.ids')).split("\n"))
  File.write(File.join(out, 'qmail.ids'), merged.join("\n"))
end

OLD = '/srv/subscriptions1' # old host puts files here
NEW = '/srv/subscriptions2' # new host puts files here
OUT = '/srv/subscriptions' # Whimsy expects the files here

if __FILE__ == $0
#   type = ARGV.shift
#   if type == 'qmail'
#     merge_qmail OLD, NEW, OUT
#   elsif type == 'files'
#     merge_files OLD, NEW, OUT
#   else
#     raise "Unexpected merge type: expected 'qmail' or 'files'"
#   end
end


# TODO: link this into a file update checker