Source code for thredds

import os
import shutil
import logging
import getpass
import ConfigParser
import zipfile
import re
from distutils.dir_util import copy_tree
import requests
import yaml
from lxml import etree
from clint.textui import progress
from esgf_utilities import esg_functions
from esgf_utilities import pybash
from esgf_utilities import esg_property_manager
from esgf_utilities import esg_truststore_manager
from base import esg_tomcat_manager, esg_postgres
from esgf_utilities.esg_env_manager import EnvWriter
from plumbum.commands import ProcessExecutionError


logger = logging.getLogger("esgf_logger" +"."+ __name__)
current_directory = os.path.join(os.path.dirname(__file__))

with open(os.path.join(current_directory, os.pardir, 'esg_config.yaml'), 'r') as config_file:
    config = yaml.load(config_file)

[docs]def check_thredds_version(): '''Check the MANIFEST.MF file for the Thredds version''' with open("/usr/local/tomcat/webapps/thredds/META-INF/MANIFEST.MF", "r") as manifest_file: contents = manifest_file.readlines() matcher = re.compile("Implementation-Version.*") results_list = filter(matcher.match, contents) if results_list: version_number = results_list[0].split(":")[1].strip().split("-")[0] logger.debug("(Thredds version %s)", version_number) return version_number else: print "Thredds not found on system."
[docs]def download_thredds_war(thredds_url): '''Download thredds war file from thredds_url''' print "\n*******************************" print "Downloading Thredds war file" print "******************************* \n" response = requests.get(thredds_url, stream=True) path = '/usr/local/tomcat/webapps/thredds/thredds.war' with open(path, 'wb') as thredds_war: total_length = int(response.headers.get('content-length')) for chunk in progress.bar(response.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1): if chunk: thredds_war.write(chunk) thredds_war.flush()
[docs]def create_password_hash(tomcat_user_password): '''Creates a hash for a Tomcat user's password using Tomcat's digest.sh script''' password_hash = esg_functions.call_subprocess("/usr/local/tomcat/bin/digest.sh -a SHA {tomcat_user_password}".format(tomcat_user_password=tomcat_user_password)) return password_hash["stdout"].split(":")[1].strip()
[docs]def update_tomcat_users_file(tomcat_username, password_hash, tomcat_users_file=config["tomcat_users_file"]): '''Adds a new user to the tomcat-users.xml file''' tree = etree.parse(tomcat_users_file) root = tree.getroot() updated_dnode_user = False for param in root.iter(): if param == "user" and param.get("username") == "dnode_user": param.set("password", password_hash) param.set("roles", "tdrAdmin,tdsConfig") updated_dnode_user = True if not updated_dnode_user: new_user = etree.SubElement(root, "user") new_user.set("username", tomcat_username) new_user.set("password", password_hash) new_user.set("roles", "tdrAdmin,tdsConfig") tree.write(open(tomcat_users_file, "w"), pretty_print=True, encoding='utf-8', xml_declaration=True)
[docs]def add_tomcat_user(): '''Add a user to the default Tomcat user database (tomcat-users.xml) for container-managed authentication''' print "Create user credentials\n" try: tomcat_username = esg_property_manager.get_property("tomcat.user") except ConfigParser.NoOptionError: default_user = "dnode_user" tomcat_username = raw_input("Please enter username for tomcat [{default_user}]: ".format(default_user=default_user)) or default_user valid_password = False while not valid_password: tomcat_user_password = esg_functions.get_security_admin_password() if not tomcat_user_password: tomcat_user_password = getpass.getpass("Please enter password for user, \"{tomcat_username}\" [********]: ".format(tomcat_username=tomcat_username)) if esg_functions.is_valid_password(tomcat_user_password): valid_password = True password_hash = create_password_hash(tomcat_user_password) update_tomcat_users_file(tomcat_username, password_hash)
def get_idp_peer_from_config(): node_type_list = esg_functions.get_node_type() if "INDEX" in node_type_list and not set(["idp", "data", "compute"]).issubset(node_type_list): try: esgf_idp_peer = esg_property_manager.get_property("esgf.idp.peer") except ConfigParser.NoOptionError: default_idp_peer = esg_functions.get_esgf_host() esgf_idp_peer = raw_input("Please specify your IDP peer node's FQDN [{}]: ".format(default_idp_peer)) or default_idp_peer return esgf_idp_peer
[docs]def select_idp_peer(esgf_idp_peer=None): '''called during setup_tds or directly by --set-idp-peer | --set-admin-peer flags''' if not esgf_idp_peer: esgf_idp_peer = get_idp_peer_from_config() esgf_idp_peer_name = esgf_idp_peer.upper() myproxy_endpoint = esgf_idp_peer esgf_host = esg_functions.get_esgf_host() # print "Selection: [${choice}] source: ${esgf_host_ip} dest: ${esgf_idp_peer_name}:${esgf_idp_peer}" if esgf_host != esgf_idp_peer: print ''' ---------------------------------------------------------------------- The IDP selected must share at least one of the peer group(s) [${node_peer_group}] that this node is a member of! run: esg-node --federation-sanity-check ${esgf_idp_peer} for confirmation. ----------------------------------------------------------------------''' if esgf_host != myproxy_endpoint: esg_truststore_manager.install_peer_node_cert(myproxy_endpoint) esg_property_manager.set_property("esgf_idp_peer_name", esgf_idp_peer_name) esg_property_manager.set_property("esgf_idp_peer", esgf_idp_peer) esg_property_manager.set_property("myproxy_endpoint", myproxy_endpoint) default_myproxy_port = 7512 esg_property_manager.set_property("myproxy_port", default_myproxy_port) write_tds_env()
[docs]def write_tds_env(): '''Write thredds info to /etc/esg.env''' EnvWriter.export("ESGF_IDP_PEER_NAME", esg_property_manager.get_property("esgf_idp_peer_name")) EnvWriter.export("ESGF_IDP_PEER", esg_property_manager.get_property("esgf_idp_peer"))
[docs]def update_mail_admin_address(): '''Updates mail_admin_address in threddsConfig.xml''' try: mail_admin_address = esg_property_manager.get_property("mail.admin.address") except ConfigParser.NoOptionError: return else: esg_functions.replace_string_in_file('/esg/content/thredds/threddsConfig.xml', "support@my.group", mail_admin_address)
[docs]def esgsetup_thredds(): '''Configures Thredds with esgsetup''' os.environ["UVCDAT_ANONYMOUS_LOG"] = "no" try: index_peer = esg_property_manager.get_property("esgf.index.peer") except ConfigParser.NoOptionError: #default peer is yourself default_peer = esg_functions.get_esgf_host() index_peer = raw_input("Enter the name of the Index Peer Node: [{}]".format(default_peer)) or default_peer security_admin_password = esg_functions.get_security_admin_password() esgsetup_options = ["--config", "--minimal-setup", "--thredds", "--publish", "--gateway", index_peer, "--thredds-password", security_admin_password] try: esg_functions.call_binary("esgsetup", esgsetup_options) except ProcessExecutionError, err: logger.error("esgsetup_thredds failed") logger.error(err) raise
[docs]def copy_public_directory(): '''HACK ALERT!! For some reason the public directory does not respect thredds' tds.context.root.path property... So have to manually move over this directory to avert server not starting! -gavin''' content_dir = os.path.join("{thredds_content_dir}".format(thredds_content_dir=config["thredds_content_dir"]), "thredds") if not os.path.isdir(content_dir): pybash.mkdir_p(content_dir) public_dir = "{}/webapps/thredds/WEB-INF/altContent/startup/public".format(config["tomcat_install_dir"]) try: copy_tree(public_dir, content_dir) except OSError: raise tomcat_user = esg_functions.get_user_id("tomcat") tomcat_group = esg_functions.get_group_id("tomcat") esg_functions.change_ownership_recursive(config["thredds_content_dir"], tomcat_user, tomcat_group)
[docs]def verify_thredds_credentials(thredds_ini_file="/esg/config/esgcet/esg.ini", tomcat_users_file=config["tomcat_users_file"]): '''Verifies that Thredds credentials in /esg/config/esgcet/esg.ini matches /esg/config/tomcat/tomcat-users.xml''' print "Inspecting tomcat... " tree = etree.parse(tomcat_users_file) root = tree.getroot() user_element = root.find("user") tomcat_username = user_element.get("username") tomcat_password_hash = user_element.get("password") print "Inspecting publisher... " thredds_username = esg_property_manager.get_property("thredds.username", property_file=thredds_ini_file, section_name="DEFAULT") thredds_password = esg_property_manager.get_property("thredds.password", property_file=thredds_ini_file, section_name="DEFAULT") thredds_password_hash = create_password_hash(thredds_password) print "Checking username... " logger.debug("tomcat_username: %s", tomcat_username) logger.debug("thredds_username: %s", thredds_username) if tomcat_username != thredds_username: print "The user_name property in {tomcat_users_file} doesn't match the user_name in {thredds_ini_file}".format(tomcat_users_file=tomcat_users_file, thredds_ini_file=thredds_ini_file) raise Exception print "Checking password... " logger.debug("tomcat_password_hash: %s", tomcat_password_hash) logger.debug("thredds_password_hash: %s", thredds_password_hash) if tomcat_password_hash != thredds_password_hash: print "The password property in {tomcat_users_file} doesn't match the password in {thredds_ini_file}".format(tomcat_users_file=tomcat_users_file, thredds_ini_file=thredds_ini_file) raise Exception print "Verified Thredds crendentials" return True
[docs]def copy_jar_files(esg_dist_url): '''TDS jars necessary to support ESGF security filters some jars are retrieved from the ESGF repository other jars are copied from the unpacked ORP or NM distributions''' esg_functions.download_update("/usr/local/tomcat/webapps/thredds/WEB-INF/lib/jdom-legacy-1.1.3.jar", "{esg_dist_url}/filters/jdom-legacy-1.1.3.jar".format(esg_dist_url=esg_dist_url)) esg_functions.download_update("/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-httpclient-3.1.jar", "{esg_dist_url}/filters/commons-httpclient-3.1.jar".format(esg_dist_url=esg_dist_url)) esg_functions.download_update("/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-lang-2.6.jar", "{esg_dist_url}/filters/commons-lang-2.6.jar".format(esg_dist_url=esg_dist_url))
[docs]def copy_xml_files(): '''Copy Thredds configuration xmls files into proper location on server''' shutil.copyfile(os.path.join(current_directory, "thredds_conf/tomcat-users.xml"), "{}/tomcat-users.xml".format(config["tomcat_conf_dir"])) pybash.mkdir_p("{tomcat_conf_dir}/Catalina/localhost".format(tomcat_conf_dir=config["tomcat_conf_dir"])) shutil.copyfile(os.path.join(current_directory, "thredds_conf/tomcat-thredds.xml"), "{}/Catalina/localhost/thredds.xml".format(config["tomcat_conf_dir"])) # TDS configuration root pybash.mkdir_p(os.path.join(config["thredds_content_dir"], "thredds")) # TDS memory configuration shutil.copyfile(os.path.join(current_directory, "thredds_conf/threddsConfig.xml"), "/esg/content/thredds/threddsConfig.xml") # ESGF root catalog shutil.copyfile(os.path.join(current_directory, "thredds_conf/catalog.xml"), "/esg/content/thredds/catalog.xml-esgcet") tomcat_user_id = esg_functions.get_user_id("tomcat") tomcat_group_id = esg_functions.get_group_id("tomcat") shutil.copyfile(os.path.join(current_directory, "thredds_conf/thredds.web.xml"), "/usr/local/tomcat/webapps/thredds/WEB-INF/web.xml") os.chown("/usr/local/tomcat/webapps/thredds/WEB-INF/web.xml", tomcat_user_id, tomcat_group_id) pybash.mkdir_p("/esg/content/thredds/esgcet") # TDS customized applicationContext.xml file with ESGF authorizer shutil.copyfile(os.path.join(current_directory, "thredds_conf/applicationContext.xml"), "/usr/local/tomcat/webapps/thredds/WEB-INF/applicationContext.xml") os.chown("{}/tomcat-users.xml".format(config["tomcat_conf_dir"]), tomcat_user_id, tomcat_group_id) # TDS customized logging (uses DEBUG) shutil.copyfile(os.path.join(current_directory, "thredds_conf/log4j2.xml"), "/usr/local/tomcat/webapps/thredds/WEB-INF/classes/log4j2.xml")
[docs]def write_tds_install_log(): '''Write thredds info to install manifest''' thredds_version = check_thredds_version() thredds_install_dir = os.path.join("{}".format(config["tomcat_install_dir"]), "webapps", "thredds") esg_functions.write_to_install_manifest("webapp:thredds", thredds_install_dir, thredds_version) esgf_host = esg_functions.get_esgf_host() esg_property_manager.set_property("thredds_service_endpoint", "http://{}/thredds".format(esgf_host)) esg_property_manager.set_property("thredds_service_app_home", "{}/webapps/thredds".format(config["tomcat_install_dir"]))
[docs]def setup_thredds(): '''Install Thredds''' print "\n*******************************" print "Setting up Thredds" print "******************************* \n" if os.path.isdir("/usr/local/tomcat/webapps/thredds"): try: thredds_install = esg_property_manager.get_property("update.thredds") except ConfigParser.NoOptionError: thredds_install = raw_input("Existing Thredds installation found. Do you want to continue with the Thredds installation [y/N]: ") or "no" if thredds_install.lower() in ["no", "n"]: print "Using existing Thredds installation. Skipping setup." return esg_tomcat_manager.stop_tomcat() pybash.mkdir_p("/usr/local/tomcat/webapps/thredds") esg_dist_url = esg_property_manager.get_property("esg.dist.url") thredds_url = "{}/thredds/5.0/{}/thredds.war".format(esg_dist_url, config["tds_version"]) download_thredds_war(thredds_url) with pybash.pushd("/usr/local/tomcat/webapps/thredds"): with zipfile.ZipFile("/usr/local/tomcat/webapps/thredds/thredds.war", 'r') as thredds_war_file: thredds_war_file.extractall() os.remove("thredds.war") tomcat_user_id = esg_functions.get_tomcat_user_id() tomcat_group_id = esg_functions.get_tomcat_group_id() esg_functions.change_ownership_recursive("/usr/local/tomcat/webapps/thredds", tomcat_user_id, tomcat_group_id) copy_xml_files() add_tomcat_user() select_idp_peer() copy_public_directory() update_mail_admin_address() copy_jar_files(esg_dist_url) # change ownership of content directory tomcat_user_id = esg_functions.get_tomcat_user_id() tomcat_group_id = esg_functions.get_tomcat_group_id() esg_functions.change_ownership_recursive("/esg/content/thredds/", tomcat_user_id, tomcat_group_id) # change ownership of source directory esg_functions.change_ownership_recursive("/usr/local/webapps/thredds", tomcat_user_id, tomcat_group_id) #restart tomcat to put modifications in effect. esg_tomcat_manager.stop_tomcat() esg_tomcat_manager.start_tomcat() esg_postgres.start_postgres() esgsetup_thredds() write_tds_install_log()
# verify_thredds_credentials() # cleanup # shutil.rmtree("/usr/local/tomcat/webapps/esgf-node-manager/")
[docs]def tds_startup_hook(): '''Prepares thredds to start''' print "TDS (THREDDS) Startup Hook: Setting permissions... " esg_functions.change_ownership_recursive(config["thredds_content_dir"], uid=esg_functions.get_user_id("tomcat"))
[docs]def main(): '''Main function''' setup_thredds()
if __name__ == '__main__': main()