fabfile.py

"""Main Fabric deployment file for CloudBioLinux distribution.

This installs a standard set of useful biological applications on a remote
server. It is designed for bootstrapping a machine from scratch, as with new
Amazon EC2 instances.

Usage:

    fab -H hostname -i private_key_file install_biolinux

which will call into the 'install_biolinux' method below. See the README for
more examples.

Requires:
    Fabric http://docs.fabfile.org
    PyYAML http://pyyaml.org/wiki/PyYAMLDocumentation
"""
import os
import sys
from datetime import datetime

from fabric.main import load_settings
from fabric.api import *
from fabric.contrib.files import *
import yaml

# use local cloudbio directory
for to_remove in [p for p in sys.path if p.find("cloudbiolinux-") > 0]:
    sys.path.remove(to_remove)
sys.path.append(os.path.dirname(__file__))
import cloudbio

from cloudbio.edition import _setup_edition
from cloudbio.distribution import _setup_distribution_environment
from cloudbio.utils import _setup_logging, _update_biolinux_log
from cloudbio.cloudman import _cleanup_ec2
from cloudbio.cloudbiolinux import _cleanup_space
from cloudbio.custom.shared import _make_tmp_dir
from cloudbio.package.shared import _yaml_to_packages
from cloudbio.package.deb import (_apt_packages, _add_apt_gpg_keys,
                                  _setup_apt_automation, _setup_apt_sources)
from cloudbio.package.rpm import (_yum_packages, _setup_yum_bashrc,
                                  _setup_yum_sources)
from cloudbio.package.nix import _setup_nix_sources, _nix_packages

# ## Utility functions for establishing our build environment

def _parse_fabricrc():
    """Defaults from fabricrc.txt file; loaded if not specified at commandline.
    """
    # ## General setup
    env.config_dir = os.path.join(os.path.dirname(__file__), "config")

    if not env.has_key("distribution") and not env.has_key("system_install"):
        env.logger.info("Reading default fabricrc.txt")
        config_file = os.path.join(env.config_dir, "fabricrc.txt")
        if os.path.exists(config_file):
            env.update(load_settings(config_file))
    else:
        env.logger.warn("Skipping fabricrc.txt as distribution is already defined")

def _create_local_paths():
    """Expand any paths defined in terms of shell shortcuts (like ~).
    """
    with settings(hide('warnings', 'running', 'stdout', 'stderr'),
                  warn_only=True):
        # This is the first point we call into a remote host - make sure
        # it does not fail silently by calling a dummy run
        env.logger.info("Now, testing connection to host...")
        test = run("pwd")
        # If there is a connection failure, the rest of the code is (sometimes) not
        # reached - for example with Vagrant the program just stops after above run
        # command.
        if test != None:
          env.logger.info("Connection to host appears to work!")
        else:
          raise NotImplementedError("Connection to host failed")
        env.logger.debug("Expand paths")
        if env.has_key("local_install"):
            if not exists(env.local_install):
                run("mkdir -p %s" % env.local_install)
            with cd(env.local_install):
                result = run("pwd")
                env.local_install = result

def _setup_flavor(flavor, environment=None):
    """Setup flavor
    """
    if not flavor:
        flavor = env.get("flavor", None)
    if not environment:
        environment = env.get("environment", None)
    if environment:
        env.environment = environment
        env.logger.info("Environment %s" % env.environment)
    if flavor:
        # import a flavor defined through parameters flavor and flavor_path
        flavor_path = env.get("flavor_path", None)
        if flavor_path == None:
          raise ImportError("You need to define the flavor_path for %s!" % flavor)
        # Add path for flavors
        sys.path.append(os.path.join(os.path.dirname(__file__), "contrib", "flavor"))
        env.logger.info("Flavor %s loaded from %s" % (flavor, flavor_path))
        try:
            mod = __import__(flavor_path, fromlist=[flavor])
        except ImportError:
            raise ImportError("Failed to import %s" % flavor)
    else:
        # import default Flavor
        from cloudbio.flavor import Flavor
    env.logger.info("This is a %s" % env.flavor.name)

# ### Shared installation targets for all platforms

def install_biolinux(target=None, packagelist=None, flavor=None, environment=None,
        pkg_config_file_path=None):
    """
    Main entry point for installing BioLinux on a remote server.

    ``packagelist`` should point to a top level file (eg, ``main.yaml``) listing
    all the package categories that should be installed. This allows a different
    package list and/or use of Flavor. So you can say::

      install_biolinux:packagelist=contrib/mylist/main.yaml,flavor=specialflavor

    ``pkg_config_file_path`` can be used to specify a path where a custom
    ``packages.yaml`` and ``packages-[dist].yaml`` are located, allowing fine-
    grained (i.e., individual package) customization. Otherwise, default
    to ``./contrib`` where the CBL files are defined.

    Both ``packagelist`` and ``flavor``, as well as the Edition, can also be
    passed in through the ``fabricrc`` file.

    target can also be supplied on the fab CLI. Special targets are:

      - packages     Install distro packages
      - custom       Install custom packages
      - libraries    Install programming language libraries
      - post_install Setup CloudMan, FreeNX and other system services
      - cleanup      Remove downloaded files and prepare images for AMI builds

    ``environment`` allows adding additional information on the command line -
    usually for defining environments, for example ``environment=testing``, or
    ``environment=production``, will set the deployment environment and tune
    post-installation settings.
    """
    _setup_logging(env)
    time_start = _print_time_stats("Config", "start")
    _check_fabric_version()
    _parse_fabricrc()
    _setup_edition(env)
    _setup_flavor(flavor, environment)
    _setup_distribution_environment() # get parameters for distro, packages etc.
    _create_local_paths()
    env.logger.debug("Meta-package list is '%s'" % packagelist)
    env.logger.debug("File path for explicit packages is '%s'" % pkg_config_file_path)
    env.logger.debug("Target is '%s'" % target)
    pkg_install, lib_install = _read_main_config(packagelist) # read main yaml
    if target is None or target == "packages":
        if env.distribution in ["debian", "ubuntu"]:
            _setup_apt_sources()
            _setup_apt_automation()
            _add_apt_gpg_keys()
            _apt_packages(pkg_install, pkg_config_file_path=pkg_config_file_path)
        elif env.distribution in ["centos", "scientificlinux"]:
            _setup_yum_sources()
            _yum_packages(pkg_install)
            _setup_yum_bashrc()
        else:
            raise NotImplementedError("Unknown target distribution")
        if env.nixpkgs: # ./doc/nixpkgs.md
            _setup_nix_sources()
            _nix_packages(pkg_install)
        _update_biolinux_log(env, target, flavor)
    if target is None or target == "custom":
        _custom_installs(pkg_install)
    if target is None or target == "libraries":
        _do_library_installs(lib_install)
    if target is None or target == "post_install":
        env.edition.post_install(pkg_install=pkg_install)
        env.flavor.post_install()
    if target is None or target == "cleanup":
        _cleanup_space(env)
        if env.has_key("is_ec2_image") and env.is_ec2_image.upper() in ["TRUE", "YES"]:
            _cleanup_ec2(env)
    _print_time_stats("Config", "end", time_start)

def _print_time_stats(action, event, prev_time=None):
    """ A convenience method for displaying time event during configuration.
    
    :type action: string
    :param action: Indicates type of action (eg, Config, Lib install, Pkg install)
    
    :type event: string
    :param event: The monitoring event (eg, start, stop)
    
    :type prev_time: datetime
    :param prev_time: A timeststamp of a previous event. If provided, duration between
                      the time the method is called and the time stamp is included in
                      the printout
                      
    :rtype: datetime
    :return: A datetime timestamp of when the method was called
    """
    time = datetime.utcnow()
    s = "{0} {1} time: {2}".format(action, event, time)
    if prev_time: s += "; duration: {0}".format(str(time-prev_time))
    env.logger.info(s)
    return time

def _check_fabric_version():
    """Checks for fabric version installed
    """
    version = env.version
    if int(version.split(".")[0]) < 1:
        raise NotImplementedError("Please install fabric version 1 or higher")

def _custom_installs(to_install):
    if not exists(env.local_install):
        run("mkdir -p %s" % env.local_install)
    pkg_config = os.path.join(env.config_dir, "custom.yaml")
    packages, pkg_to_group = _yaml_to_packages(pkg_config, to_install)
    for p in env.flavor.rewrite_config_items("custom", packages):
        install_custom(p, True, pkg_to_group)

def install_custom(p, automated=False, pkg_to_group=None):
    """Install a single custom package by name.
    This method fetches names from custom.yaml that delegate to a method
    in the custom/name.py program. Alternatively, if a program install method is
    defined in approapriate package, it will be called directly (see param p).
    
    Usage: fab [-i key] [-u user] -H host install_custom:program_name
    
    :type p:  string
    :param p: A name of a custom program to install. This has to be either a name
              that is listed in custom.yaml as a subordinate to a group name or a
              program name whose install method is defined in either cloudbio or
              custom packages (eg, install_cloudman).
    
    :type automated:  bool
    :param automated: If set to True, the environment is not loaded and reading of
                      the custom.yaml is skipped.
    """
    _setup_logging(env)
    p = p.lower() # All packages are listed in custom.yaml are in lower case
    time_start = _print_time_stats("Custom install for '{0}'".format(p), "start")
    if not automated:
        _parse_fabricrc()
        _setup_edition(env)
        _setup_distribution_environment()
        _create_local_paths()
        pkg_config = os.path.join(env.config_dir, "custom.yaml")
        packages, pkg_to_group = _yaml_to_packages(pkg_config, None)

    try:
        env.logger.debug("Import %s" % p)
        # Allow direct calling of a program install method, even if the program
        # is not listed in the custom list (ie, not contained as a key value in
        # pkg_to_group). For an example, see 'install_cloudman' or use p=cloudman.
        mod_name = pkg_to_group[p] if p in pkg_to_group else p
        mod = __import__("cloudbio.custom.%s" % mod_name,
                         fromlist=["cloudbio", "custom"])
    except ImportError:
        raise ImportError("Need to write a %s module in custom." %
                pkg_to_group[p])
    replace_chars = ["-"]
    try:
        for to_replace in replace_chars:
            p = p.replace(to_replace, "_")
        fn = getattr(mod, "install_%s" % p)
    except AttributeError:
        raise ImportError("Need to write a install_%s function in custom.%s"
                % (p, pkg_to_group[p]))
    fn(env)
    _print_time_stats("Custom install for '%s'" % p, "end", time_start)

def _read_main_config(yaml_file=None):
    """Pull a list of groups to install based on our main configuration YAML.

    Reads 'main.yaml' and returns packages and libraries
    """
    if yaml_file is None:
        yaml_file = os.path.join(env.config_dir, "main.yaml")
    with open(yaml_file) as in_handle:
        full_data = yaml.load(in_handle)
    packages = full_data['packages']
    packages = packages if packages else []
    libraries = full_data['libraries']
    libraries = libraries if libraries else []
    env.logger.info("Meta-package information from {2}\n- Packages: {0}\n- Libraries: "
            "{1}".format(",".join(packages), ",".join(libraries), yaml_file))
    return packages, sorted(libraries)

# ### Library specific installation code

def _r_library_installer(config):
    """Install R libraries using CRAN and Bioconductor.
    """
    # Create an Rscript file with install details.
    out_file = "install_packages.R"
    if exists(out_file):
        run("rm -f %s" % out_file)
    run("touch %s" % out_file)
    repo_info = """
    cran.repos <- getOption("repos")
    cran.repos["CRAN" ] <- "%s"
    options(repos=cran.repos)
    source("%s")
    """ % (config["cranrepo"], config["biocrepo"])
    append(out_file, repo_info)
    install_fn = """
    repo.installer <- function(repos, install.fn) {
      update.or.install <- function(pname) {
        if (pname %in% installed.packages())
          update.packages(lib.loc=c(pname), repos=repos, ask=FALSE)
        else
          install.fn(pname)
      }
    }
    """
    append(out_file, install_fn)
    std_install = """
    std.pkgs <- c(%s)
    std.installer = repo.installer(cran.repos, install.packages)
    lapply(std.pkgs, std.installer)
    """ % (", ".join('"%s"' % p for p in config['cran']))
    append(out_file, std_install)
    bioc_install = """
    bioc.pkgs <- c(%s)
    bioc.installer = repo.installer(biocinstallRepos(), biocLite)
    lapply(bioc.pkgs, bioc.installer)
    """ % (", ".join('"%s"' % p for p in config['bioc']))
    append(out_file, bioc_install)
    final_update = """
    update.packages(repos=biocinstallRepos(), ask=FALSE)
    update.packages(ask=FALSE)
    """
    append(out_file, final_update)
    # run the script and then get rid of it
    env.safe_sudo("Rscript %s" % out_file)
    run("rm -f %s" % out_file)

def _python_library_installer(config):
    """Install python specific libraries using easy_install.
    """
    version_ext = "-%s" % env.python_version_ext if env.python_version_ext else ""
    env.safe_sudo("easy_install%s -U pip" % version_ext)
    for pname in env.flavor.rewrite_config_items("python", config['pypi']):
        env.safe_sudo("easy_install%s -U %s" % (version_ext, pname))
        # Use pip when it doesn't re-download even if latest package installed
        # https://bitbucket.org/ianb/pip/issue/13/upgrade-always-downloads-most-recent
        #sudo("pip%s install -U %s" % (version_ext,  pname))

def _ruby_library_installer(config):
    """Install ruby specific gems.
    """
    gem_ext = getattr(env, "ruby_version_ext", "")
    def _cur_gems():
        with settings(
                hide('warnings', 'running', 'stdout', 'stderr')):
            gem_info = run("gem%s list --no-versions" % gem_ext)
        return [l.rstrip("\r") for l in gem_info.split("\n") if l.rstrip("\r")]
    installed = _cur_gems()
    for gem in env.flavor.rewrite_config_items("ruby", config['gems']):
        # update current gems only to check for new installs
        if gem not in installed:
            installed = _cur_gems()
        if gem in installed:
            env.safe_sudo("gem%s update %s" % (gem_ext, gem))
        else:
            env.safe_sudo("gem%s install %s" % (gem_ext, gem))

def _perl_library_installer(config):
    """Install perl libraries from CPAN with cpanminus.
    """
    with _make_tmp_dir() as tmp_dir:
        with cd(tmp_dir):
            run("wget --no-check-certificate -O cpanm "
                "https://raw.github.com/miyagawa/cpanminus/master/cpanm")
            run("chmod a+rwx cpanm")
            env.safe_sudo("mv cpanm %s/bin" % env.system_install)
    sudo_str = "--sudo" if env.use_sudo else ""
    for lib in env.flavor.rewrite_config_items("perl", config['cpan']):
        # Need to hack stdin because of some problem with cpanminus script that
        # causes fabric to hang
        # http://agiletesting.blogspot.com/2010/03/getting-past-hung-remote-processes-in.html
        run("cpanm %s --skip-installed --notest %s < /dev/null" % (sudo_str, lib))

def _clojure_library_installer(config):
    """Install clojure libraries using cljr.
    """
    for lib in config['cljr']:
        run("cljr install %s" % lib)

def _haskell_library_installer(config):
    """Install haskell libraries using cabal.
    """
    run("cabal update")
    for lib in config["cabal"]:
        sudo_str = "--root-cmd=sudo" if env.use_sudo else ""
        run("cabal install %s --global %s" % (sudo_str, lib))

lib_installers = {
    "r-libs" : _r_library_installer,
    "python-libs" : _python_library_installer,
    "ruby-libs" : _ruby_library_installer,
    "perl-libs" : _perl_library_installer,
    "clojure-libs": _clojure_library_installer,
    "haskell-libs": _haskell_library_installer,
    }

def install_libraries(language):
    """High level target to install libraries for a specific language.
    """
    _setup_logging(env)
    _check_fabric_version()
    _parse_fabricrc()
    _setup_edition(env)
    _setup_flavor(None)
    _setup_distribution_environment()
    _create_local_paths()
    _do_library_installs(["%s-libs" % language])

def _do_library_installs(to_install):
    for iname in to_install:
        yaml_file = os.path.join(env.config_dir, "%s.yaml" % iname)
        with open(yaml_file) as in_handle:
            config = yaml.load(in_handle)
        lib_installers[iname](config)