import os
import logging
import shutil
import ConfigParser
import OpenSSL
import stat
import glob
import psutil
import yaml
from esgf_utilities import esg_functions
from esgf_utilities import pybash
from esgf_utilities import esg_property_manager
from esgf_utilities import esg_version_manager
from esgf_utilities import esg_cert_manager
from esgf_utilities.esg_exceptions import InvalidNodeTypeError
from base import esg_tomcat_manager
from base import esg_postgres
from idp_node import gridftp
from idp_node import myproxy
from esgf_utilities.esg_env_manager import EnvWriter
from plumbum.commands import ProcessExecutionError
logger = logging.getLogger("esgf_logger" +"."+ __name__)
current_directory = os.path.join(os.path.dirname(__file__))
with open(os.path.join(current_directory, os.pardir, 'esg_config.yaml'), 'r') as config_file:
config = yaml.load(config_file)
def check_for_globus_installation(globus_version, installation_type):
if os.access("/usr/bin/globus-version", os.X_OK):
print "Detected an existing Globus installation"
print "Checking for Globus {}".format(globus_version)
installed_globus_version = esg_functions.call_binary("/usr/bin/globus-version")
if esg_version_manager.compare_versions(installed_globus_version, globus_version+".0"):
if installation_type == "DATA" and os.path.exists("/usr/bin/globus-connect-server-io-setup"):
print "GridFTP already installed"
print "Globus version appears sufficiently current"
return True
if installation_type == "IDP" and os.path.exists("/usr/bin/globus-connect-server-id-setup"):
print "MyProxy already installed"
print "Globus version appears sufficiently current"
return True
[docs]def setup_globus(installation_type):
'''
Globus Toolkit -> MyProxy (client) & GridFTP (server)
Takes arg <selection bit vector>
The rest of the args are the following...
for data-node configuration (GridFTP stuff): ["bdm"|"end-user"] see esg-globus script
for idp configuration (MyProxy stuff): [gen-self-cert] <dir> | <regen-simpleca> [fetch-certs|gen-self-cert|keep-certs] | ["install"|"update"]'''
logger.debug("setup_globus for installation type: %s", installation_type)
globus_version = "6.0"
if check_for_globus_installation(globus_version, installation_type):
try:
setup_globus_answer = esg_property_manager.get_property("update.globus")
if not setup_globus_answer:
raise ConfigParser.NoOptionError
except ConfigParser.NoOptionError:
setup_globus_answer = raw_input(
"Do you want to continue with the Globus installation and setup? [y/N]: ") or "N"
if setup_globus_answer.lower().strip() in ["no", 'n']:
logger.info("Skipping Globus installation. Using existing Globus version")
return
globus_location = "/usr/local/globus"
pybash.mkdir_p(config["workdir"])
with pybash.pushd(config["workdir"]):
directive = "notype"
if installation_type == "DATA":
logger.info("Globus Setup for Data-Node... (GridFTP server) ")
directive = "datanode"
setup_globus_services(directive)
write_globus_env(globus_location)
pybash.touch(os.path.join(globus_location,"esg_esg-node_installed"))
if installation_type == "IDP":
logger.info("Globus Setup for Index-Node... (MyProxy server)")
directive = "gateway"
setup_mode = "install"
setup_globus_services(directive)
write_globus_env(globus_location)
pybash.touch(os.path.join(globus_location, "esg_esg-node_installed"))
[docs]def write_globus_env(globus_location):
'''Write globus properties to /etc/esg.env'''
EnvWriter.export("GLOBUS_LOCATION", globus_location)
[docs]def start_globus(installation_type):
'''Starts the globus services by delegating out to esg-globus script
arg1 selection bit vector ($sel)
args* (in the context of "data" node -> ["bdm"|"end-user"])'''
if installation_type == "DATA":
directive = "datanode"
start_globus_services(directive)
if installation_type == "IDP":
directive = "gateway"
start_globus_services(directive)
[docs]def stop_globus(installation_type):
'''Stops the globus services'''
if installation_type == "DATA":
stop_globus_services("datanode")
elif installation_type == "IDP":
stop_globus_services("gateway")
else:
raise InvalidNodeTypeError("Globus is not installed on a {} node type".format(installation_type))
#--------------------------------------------------------------
# PROCEDURE
#--------------------------------------------------------------
[docs]def setup_globus_services(config_type):
'''arg1 - config_type ("datanode" | "gateway" ["install"|"update"])'''
print "*******************************"
print "Setting up Globus... (config type: {})".format(config_type)
print "*******************************"
globus_sys_acct = "globus"
logger.debug("setup_globus_services for %s", config_type)
globus_location = "/usr/local/globus"
pybash.mkdir_p(os.path.join(globus_location, "bin"))
if config_type == "datanode":
print "*******************************"
print "Setting up ESGF Globus GridFTP Service(s)"
print "*******************************"
create_globus_account(globus_sys_acct)
_install_globus(config_type)
gridftp.setup_gcs_io("firstrun")
gridftp.setup_gridftp_metrics_logging()
gridftp.config_gridftp_server(globus_sys_acct)
gridftp.config_gridftp_metrics_logging()
if os.path.exists("/usr/sbin/globus-gridftp-server"):
esg_property_manager.set_property("gridftp_app_home", "/usr/sbin/globus-gridftp-server")
elif config_type == "gateway":
print "*******************************"
print "Setting up The ESGF Globus MyProxy Services"
print "*******************************"
_install_globus(config_type)
myproxy.setup_gcs_id("firstrun")
myproxy.config_myproxy_server(globus_location)
else:
print "You must provide a configuration type arg [datanode | gateway]"
return
def start_globus_services(config_type):
print "Starting Globus services for {}".format(config_type)
if config_type == "datanode":
gridftp.start_gridftp_server()
esg_property_manager.set_property("gridftp_endpoint", "gsiftp://{}".format(esg_functions.get_esgf_host()))
elif config_type == "gateway":
myproxy.start_myproxy_server()
else:
print "You must provide a configuration type arg [datanode | gateway]"
return
[docs]def stop_globus_services(config_type):
'''Stop globus'''
print "stop_globus_services for {}".format(config_type)
if config_type == "datanode":
gridftp.stop_gridftp_server()
elif config_type == "gateway":
myproxy.stop_myproxy_server()
else:
print "You must provide a configuration type arg [datanode | gateway]"
return
[docs]def check_for_existing_globus_rpm_packages():
'''Check if globus rpm is already installed'''
rpm_packages = esg_functions.call_binary("rpm", ["-qa"]).split("\n")
globus_packages = [package for package in rpm_packages if "globus" in package]
return globus_packages
#--------------------------------------------------------------
# GLOBUS INSTALL (subset)
#--------------------------------------------------------------
# All methods below this point should be considered "private" functions
[docs]def install_globus_rpm():
'''Install globus rpm repo'''
globus_connect_server_file = "globus-connect-server-repo-latest.noarch.rpm"
globus_connect_server_url = "http://toolkit.globus.org/ftppub/globus-connect-server/globus-connect-server-repo-latest.noarch.rpm"
esg_functions.download_update(globus_connect_server_file, globus_connect_server_url)
esg_functions.call_binary("rpm", ["--import", "http://www.globus.org/ftppub/globus-connect-server/RPM-GPG-KEY-Globus"])
esg_functions.call_binary("rpm", ["-i", "globus-connect-server-repo-latest.noarch.rpm"])
def _install_globus(config_type):
if config_type == "datanode":
globus_type = "globus-connect-server-io"
elif config_type == "gateway":
globus_type = "globus-connect-server-id"
else:
print "You must provide a configuration type arg [datanode | gateway]"
return
globus_workdir = os.path.join(config["workdir"], "extra", "globus")
pybash.mkdir_p(globus_workdir)
current_mode = os.stat(globus_workdir)
# chmod a+rw $globus_workdir
os.chmod(globus_workdir, current_mode.st_mode | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
with pybash.pushd(globus_workdir):
# Setup Globus RPM repo
if not check_for_existing_globus_rpm_packages():
install_globus_rpm()
# Install Globus and ESGF RPMs
esg_functions.call_binary("yum", ["-y", "install", globus_type])
esg_functions.call_binary("yum", ["-y", "update", globus_type])
if globus_type == "globus-connect-server-io":
try:
esg_functions.call_binary("yum", ["-y", "install", "--nogpgcheck", "globus-authz-esgsaml-callout", "globus-gaa", "globus-adq", "customgsiauthzinterface"])
esg_functions.call_binary("yum", ["-y", "update", "--nogpgcheck", "globus-authz-esgsaml-callout", "globus-gaa", "globus-adq", "customgsiauthzinterface"])
except ProcessExecutionError, error:
logger.error(error)
pass
else:
#TODO: remove --nogpgcheck check when packages are properly signed
esg_functions.call_binary("yum", ["-y", "install", "--nogpgcheck", "mhash", "pam-pgsql"])
esg_functions.call_binary("yum", ["-y", "update", "--nogpgcheck", "mhash", "pam-pgsql"])
############################################
# Utility Functions
############################################
[docs]def create_globus_account(globus_sys_acct):
'''Create the system account for globus to run as.'''
esg_functions.add_unix_group(globus_sys_acct)
globus_sys_acct_passwd = esg_functions.get_security_admin_password()
useradd_options = ["-r", "-c", "Globus System User", "-g", "globus", "-p", globus_sys_acct_passwd, "-s", "/bin/bash", globus_sys_acct]
esg_functions.add_unix_user(useradd_options)
[docs]def globus_check_certificates():
'''Check if globus certificates are valid'''
print "globus_check_certificates..."
my_cert = "/etc/grid-security/hostcert.pem"
esg_cert_manager.check_cert_expiry(my_cert)