#!/usr/local/bin/python

import os
import syslog
import re
import atexit
import sys
import datetime
from time import mktime

import isi.fs.siq as siq
from isi.fs.siq_upgrade import *
import isi.celog
from isi.celog.eventids import eventid_map
from sqlite3 import dbapi2 as sqlite

# File locations
ifsvar_tsm_path = "/ifs/.ifsvar/modules/tsm"
config_dir_path = "/ifs/.ifsvar/modules/tsm/config"
upgrade_done_file = "/ifs/.ifsvar/modules/tsm/backup/.mav-upgrade-done"
upgrade_error_file = "/ifs/.ifsvar/modules/tsm/backup/.mav-upgrade-error"
upgrade_lock_file = "/ifs/.ifsvar/modules/tsm/config/.upgrade.lock"
upgrade_lock_fd = -1
path_to_backup = "/ifs/.ifsvar/modules/tsm"
backup_path = "/ifs/.ifsvar/modules/tsm/backup"
old_policy_file = "/ifs/.ifsvar/modules/tsm/config/tsm-policies.xml"
source_record_file = "/ifs/.ifsvar/modules/tsm/config/source_record.xml"
run_path = "/ifs/.ifsvar/modules/tsm/sched/run/"
reports_base_path = "/ifs/.ifsvar/modules/tsm/sched/reports/"
reports_db_path = "/ifs/.ifsvar/modules/tsm/sched/reports/reports.db"
global_config_path = "/etc/mcp/sys/tsm.xml"
global_config_override_path = "/etc/mcp/override/tsm.xml"
bwt_path = "/etc/mcp/override/bwt.xml"
report_db_path = reports_base_path + "reports.db"
upgrade_failpoint_file = "/ifs/.ifsvar/modules/tsm/config/.mav-upgrade-failpoint"

now = datetime.datetime.now()
now_timestamp = int(mktime(now.timetuple()))
backup_file =  "/ifs/.ifsvar/modules/tsm/backup/pre-mav." + str(now_timestamp) + ".tgz"

exit_status = -1

def exiting():
	if exit_status != 0:
		os.system("touch %s" % (upgrade_error_file))
		if (os.access(upgrade_done_file, os.F_OK)):
			os.unlink(upgrade_done_file)
		syslog.syslog(syslog.LOG_NOTICE, "SIQ Upgrade failed.")
		try:
			siq_ev = isi.celog.Event(
				event_id=eventid_map["SW_SIQ_POLICY_UPGRADE"],
				severity=isi.celog.EVENT_LEVEL_CRIT,
				message='SyncIQ upgrade has failed to complete successfully.')
			siq_ev.add()
		except Exception, exc:
			syslog.syslog(syslog.LOG_ERR, "Failed to create upgrade failure alert. Error: %s" % exc)
	else:
		os.system("touch %s" % (upgrade_done_file))
		os.chmod(upgrade_done_file, 0640)
		if (os.access(upgrade_error_file, os.F_OK)):
			os.unlink(upgrade_error_file)
		syslog.syslog(syslog.LOG_NOTICE, "SIQ Upgrade completed.")
	
	#Release upgrade lock
	if upgrade_lock_fd != -1:
		os.close(upgrade_lock_fd)
		

atexit.register(exiting)
siqu = siq.SyncIQUtils()

syslog.openlog("isi_migrate")
syslog.syslog(syslog.LOG_NOTICE, "Starting SyncIQ Upgrade (Mav)...")


if (os.access(upgrade_failpoint_file, os.F_OK)):
    syslog.syslog(syslog.LOG_ERR, " ".join([
        "Detected failpoint file.  Simulating Upgrade Failure. ",
        "To prevent this from happening, remove %s." % upgrade_failpoint_file
    ]))
    sys.exit(-1)

#Grab lock on upgrade file
if (not os.access(ifsvar_tsm_path, os.F_OK)):
	#Nothing exists, we are done
	syslog.syslog(syslog.LOG_INFO, "SIQ config directory not found")
	os.system("mkdir -p %s" % backup_path)
	exit_status = 0
	sys.exit(0)

os.system("mkdir -p %s" % backup_path)
os.system("mkdir -p %s" % config_dir_path)
upgrade_lock_fd = os.open(upgrade_lock_file, os.O_EXLOCK | os.O_CREAT, 0400)
os.chmod(upgrade_lock_file, 0400)

def iterate_and_parse_policies(policies):
	synciq = siq.SyncIQ()
	removed_policy_flag = False
        
	for xml_policy in policies:
		try:
			new_pol = siq.Policy()
			new_pc = siq.PolicyConfig()
			
			#Skip any system policies
			pol_type = get_xml_data(xml_policy, "type")
			if (pol_type == "system"):
				continue
		
			#Common
			new_pol.name = get_xml_data(xml_policy, "name")
			syslog.syslog(syslog.LOG_INFO, "Upgrading policy \"%s\"" % new_pol.name)
			new_pc.description = get_xml_data(xml_policy, "desc")
			new_pol.upgrade_set_pid(get_policy_id(xml_policy))
			new_pol.action = get_action(xml_policy)
			if (new_pol.action == siq.SIQ_ACT_NOP):
				#Skip this one
				removed_policy_flag = true
				continue
			new_pc.log_level = loglevel_to_int(get_xml_data(xml_policy, "log-level"))
		
			#Scheduler
			new_pol.schedule = get_xml_data(xml_policy, "time-table")
		
			set_field_int(new_pc, "max_reports", xml_policy, "max-reports")
			set_field_int(new_pc, "rotate_report_period", xml_policy, "rotate-report-period")
		
			#Coordinator
			set_field_int(new_pc, "workers_per_node", xml_policy, "workers-per-node")
			set_field_bool(new_pc, "log_removed", xml_policy, "log-removed-files")
			set_field_bool(new_pc, "diff_sync", xml_policy, "diff-sync")
			set_field_bool(new_pc, "skip_bb_hash", xml_policy, "skip-bb-hash")
			set_field_long(new_pc, "rename_expiration", xml_policy, "rename-expiration")
			new_pc.rename_pattern = get_xml_data(xml_policy, "rename-pattern")
			set_field_bool(new_pc, "error_checking", xml_policy, "check-integrity")
			dst_node = get_xml_node(xml_policy, "dst")
			new_pc.snapshot_on_target = get_dst_snapshot_mode(dst_node)
			new_pc.snapshot_on_target_name_pattern = get_xml_data(dst_node, "snapshot-pattern")
			new_pc.snapshot_on_target_alias = get_xml_data(dst_node, "snapshot-alias")
			set_field_long(new_pc, "snapshot_on_target_expiration", xml_policy, "snapshot-expiration")
		
			#Datapath
			src_node = get_xml_node(xml_policy, "src")
			# datapath["src_cluster_name"] = get_xml_data(src_node, "clu")
			# datapath["datapath_hash"] = ""  This is handled later below
			new_pc.root_path = get_xml_data(src_node, "root_path")
			paths_node = get_xml_node(xml_policy, "paths")
			paths_children = paths_node.getElementsByTagName("path")
			for current in paths_children:
				path = current.firstChild.data
				path_type = str(current.attributes["type"].value)
				if path_type == "exclude":
					new_pc.excludes.append(path)
				else:
					new_pc.includes.append(path)
		
			new_pc.predicate = get_xml_data(xml_policy, "predicate")
			# stf-sync shipped disabled in 6.0 and later got renamed
			# to disable-stf in 6.5
			set_field_bool(new_pc, "disable_stf", xml_policy, "disable-stf")
			set_field_long(new_pc, "siq_v1_lastrun", xml_policy, "siq-v1-lastrun")
			new_pc.snapshot_on_source = get_src_snapshot_mode(src_node)
		
			#Target
			new_pc.target_cluster = get_xml_data(dst_node, "clu")
			new_pc.shared_secret = get_xml_data(dst_node, "password")
			new_pc.target_path = get_xml_data(dst_node, "path")
			new_pc.restrict_by = get_xml_data(xml_policy, "restrict-by")
			set_field_bool(new_pc, "target_restrict", xml_policy, "target-restrict")
			set_field_bool(new_pc, "force_interface", xml_policy, "force-interface")
		
			#Source Record
			new_pol.state = get_state(xml_policy)
		
			#Begin Legacy Upgrade functions
			#Calculate root_path, if it does not exist
			if (new_pc.root_path == None or new_pc.root_path == ""):
				calculate_root_path(new_pol, new_pc)
		
			#Bug 52630 - Need to look for policies with source snapshot disabled and fix them
			new_pc.snapshot_on_source = 1
		
			#If there was no rotate report set above, set the default
			if (new_pc.rotate_report_period == -1):
				new_pc.rotate_report_period = 31536000
		
			#End Legacy Upgrade functions
			
			#Save
			synciq.add_policy(new_pol, new_pc)
			
			#Update datapath_hash
			new_pol.force_update_datapath_hash()
		except Exception, exc:
			syslog.syslog(syslog.LOG_ERR, "Failed to upgrade policy %s." 
				% new_pol.name)
			raise
	
	if removed_policy_flag:
		syslog.syslog(syslog.LOG_ERR, "One or more SyncIQ Move policies were "
			"deleted during the upgrade process")
		try:
			siq_ev = isi.celog.Event(
				event_id=eventid_map["SW_SIQ_UPGRADE_ALERT"],
				severity=isi.celog.EVENT_LEVEL_CRIT,
				message='One or more SyncIQ "Move" policies were deleted '
				'during the upgrade process')
			siq_ev.add()
		except:
			syslog.syslog(syslog.LOG_ERR, "Failed to create move alert")
	
syslog.syslog(syslog.LOG_INFO, "Backing up existing SIQ configuration...")

# BACKUP EXISTING FILES + SANITY CHECK
# Determine if the upgrade needs to happen
if (os.access(upgrade_done_file, os.F_OK)):
	syslog.syslog(syslog.LOG_NOTICE, "Skipping upgrade.  Upgrade already completed.")
	exit_status = 0
	sys.exit(0)

# Backup old config and sched folders
# Mkdir backup folder
try:
	os.mkdir(backup_path)
except:
	pass

# Make a tarball of old files, but only stuff that exists
to_backup = ""
if (os.access("/ifs/.ifsvar/modules/tsm/config", os.F_OK)):
	to_backup = to_backup + "/ifs/.ifsvar/modules/tsm/config "

if (os.access("/ifs/.ifsvar/modules/tsm/sched/reports", os.F_OK)):
	to_backup = to_backup + "/ifs/.ifsvar/modules/tsm/sched/reports "

if (os.access("/ifs/.ifsvar/modules/tsm/sched/run", os.F_OK)):
	to_backup = to_backup + "/ifs/.ifsvar/modules/tsm/sched/run "

if (os.access("/etc/mcp/override/tsm.xml", os.F_OK)):
	to_backup = to_backup + "/etc/mcp/override/tsm.xml "

if (os.access("/etc/mcp/override/bwt.xml", os.F_OK)):
	to_backup = to_backup + "/etc/mcp/override/bwt.xml "

if (os.access("/etc/mcp/sys/tsm.xml", os.F_OK)):
	to_backup = to_backup + "/etc/mcp/sys/tsm.xml "

tarcmd = "tar -c -z -f %s %s > /dev/null 2>&1" % (backup_file, to_backup)
syslog.syslog(syslog.LOG_INFO, "Executing tar: \"%s\"" % (tarcmd))
os.system(tarcmd)
os.chmod(backup_file, 0640)

# Remove existing gc files in config (maybe failed upgrade?)
rmgccmd = "find %s -name '*.gc' -exec rm -rf {} \;" % (config_dir_path)
os.system(rmgccmd)

# Globals for later use
restrict_by = None
target_restrict = None
force_interface = None

# UPGRADE POLICIES
syslog.syslog(syslog.LOG_INFO, "Upgrading policies...")
pol_file = None
pol_dom = None
try:
	if (os.access(old_policy_file, os.F_OK)):
		pol_file, pol_dom = open_and_parse_xml(old_policy_file)
		nodes = pol_dom.getElementsByTagName("policy")
		if (len(nodes) >= 1):
			iterate_and_parse_policies(nodes)
		defaults = pol_dom.getElementsByTagName("defaults")
		if (len(defaults) >= 1):
			default = defaults[0]
			restrict_by = get_xml_data(default, "restrict-by")
			tr = get_xml_bool(default, "target-restrict")
			if tr:
				target_restrict = tr
			fi = get_xml_bool(default, "force-interface")
			if fi:
				force_interface = fi
	else:
		syslog.syslog(syslog.LOG_INFO, "Skipping policies.  No policies found.")
except Exception, exc:
            syslog.syslog(syslog.LOG_ERR, "Failed to upgrade policies.  Error: %s" % exc)
            sys.exit(-1)
finally:
	if pol_dom:
		#pol_dom.unlink()
		pass
	if pol_file:
		pol_file.close()

class Enum(object):
    MAP = []

    def __init__(self, key):
        self._entnum = None

        if type(key) == int:
            if 0 <= key < len(self.MAP):
                self._entnum = key
            else:
                raise IndexError("key value %u is out of range [0, %u]" %
                    (key, len(self.MAP)-1))
        elif type(key) == str:
            for i in range(0, len(self.MAP)):
                entry = self.MAP[i]
                if type(entry) in (list, tuple):
                    if key in entry:
                        self._entnum = i
                        break
                elif entry == key:
                    self._entnum = i
                if self._entnum:
                    break
            if self._entnum == None:
                raise IndexError("key '%s' was not found" % key)
        else:
            raise TypeError("'key' must be of type 'str' or 'int'")

    def __int__(self):
        return self._entnum

    def __str__(self):
        entry = self.MAP[self._entnum]
        if type(entry) in (list, tuple):
            return entry[0]
        else:
            return entry

    def __repr__(self):
        return self._entnum

    def __cmp__(self, other):
        return cmp(self._entnum, int(other))

# This map was copied from tsmcmd.py and probably needs to be kept in sync,
# or both instances need to be moved to a new shared utils file
class ActionTypeEnum(Enum):
    MAP = [
	'nop',
	['copy', 'cp'],
	['move', 'mv'],
	['remove', 'rm'],
	'sync',
	['allow_write'],
	['disable_write'],
	['resync_prep'],
	['resync_prep_domain_mark'],
	['resync_prep_restore'],
	['resync_prep_finalize'],
	['resync_sync'],
	['resync_commit'],
	['snap_revert_domain_mark'],
	['synciq_domain_mark'],
	['move', 'mv']
    ]

def iterate_and_parse_specs():
	dirs = os.listdir(run_path)
	synciq = siq.SyncIQ()
	current_policies = synciq.get_policies()
	for current in dirs:
		sdom = None
		sfile = None
		filename = run_path + current + "/spec.xml"
		if (os.access(filename, os.F_OK)):
			try:
				sfile, sdom = open_and_parse_xml(filename)
				pid = str(current.split("_")[0])
				spec = parse_spec(sdom, pid)
				spec["common"]["action"] = str(ActionTypeEnum(current_policies[pid].action))
				spec_file = filename[:-3] + "gc"
				scfg = siqu.get_gconfig_spec(spec_file)
				open_and_write_gc(scfg, spec)
			except Exception, exc:
				syslog.syslog(syslog.LOG_ERR, "Failed to upgrade specs. Error: %s" % exc)
				sys.exit(-1)
			finally:
				if sdom:
					sdom.unlink()
				if sfile:
					sfile.close()

def iterate_and_parse_running_reports():
	dirs = os.listdir(run_path)
	for current in dirs:
		rdom = None
		rfile = None
		filename = run_path + current + "/report.xml"
		if (os.access(filename, os.F_OK)):
			try:
				rfile, rdom = open_and_parse_xml(filename)
				pid = str(current.split("_")[0])
				rpt = parse_report(rdom, pid)
				rpt_file = filename[:-3] + "gc"
				rcfg = siqu.get_gconfig_report(rpt_file)
				open_and_write_gc(rcfg, rpt)
			except Exception, exc:
				syslog.syslog(syslog.LOG_ERR, "Failed to upgrade running reports. Error: %s" % exc)
				sys.exit(-1)
			finally:
				if rdom:
					rdom.unlink()
				if rfile:
					rfile.close()

# UPGRADE SPECS
syslog.syslog(syslog.LOG_INFO, "Upgrading specs...")
if (os.access(run_path, os.F_OK)):
	iterate_and_parse_specs()
else:
	syslog.syslog(syslog.LOG_INFO, "Run directory not found, skipping")

syslog.syslog(syslog.LOG_INFO, "Upgrading running reports...")
# UPGRADE RUNNING REPORTS
if (os.access(run_path, os.F_OK)):
    iterate_and_parse_running_reports()
else:
	syslog.syslog(syslog.LOG_INFO, "Run directory not found, skipping")

def iterate_and_add_reports(pid):
	syslog.syslog(syslog.LOG_INFO, "Adding reportdb entries for %s..." % pid)
	files = os.listdir(reports_base_path + pid)
	files.sort()
	legacy_jid_counter = 1
	conn = sqlite.connect(reports_db_path)
	cur = conn.cursor()
	last_report = ""
	for file in files:
		if file[0:7] == "report-" and file.find(".xml") > 0:
			filename = reports_base_path + pid + "/" + file
			timestamp = file[7:-4]
			job_id = "LEGACY-" + str(legacy_jid_counter)
			cur.execute(siq.REPORTS_INSERT_SQL % (filename, pid, job_id, long(timestamp)))
			legacy_jid_counter = legacy_jid_counter + 1
			last_report = filename
	conn.commit()
	cur.close()

def upgrade_reports():
	pids = os.listdir(reports_base_path)
	for pid in pids:
		if len(pid) == 32:
			try:
				iterate_and_add_reports(pid)
			except Exception, exc:
				syslog.syslog(syslog.LOG_ERR, "Failed to parse reports for %s. Error: %s" % (pid, exc))
				raise
				sys.exit(-1)

# When upgrading reports, we only upgrade up to the Last Known Good (LKG) report
# to avoid hanging the upgrade process for potentially hours.  Remaining reports 
# will be upgraded inline on first access.
def upgrade_lkg_reports():
	siq_obj = siq.SyncIQ()
	policies = siq_obj.get_policies()
	syslog.syslog(syslog.LOG_INFO, "Upgrading reports up to LKG...")
	for policy in policies.values():
		pid = policy.policy_id
		syslog.syslog(syslog.LOG_INFO, "Upgrading reports for %s..." % pid)
		conn = sqlite.connect(reports_db_path)
		cur = conn.cursor()
		cur.execute("SELECT filename FROM reports WHERE policy_id = '%s' ORDER BY timestamp DESC;" % pid)
		returned = cur.fetchall()
		counter = 0
		lkg = "0";
		job_id = "1"
		inc_job_id = 1
		files_to_del = []
		for current in returned:
			counter = counter + 1
			filename = current[0]
			timestamp = filename[-14:-4]
			is_success = False
			try:
				rfile, rdom = open_and_parse_xml(filename)
				report = parse_report(rdom, pid)
				gc_file = filename[:-3] + "gc"
				rgcfg = siqu.get_gconfig_report(gc_file)
				open_and_write_gc(rgcfg, report)
				state = report["summary"]["total"]["state"]
				if (state == "success") or (state == "success with failures"):
					is_success = True
					lkg = timestamp
				current_jid = report["summary"]["job_id"]
				if counter == 1 and current_jid != 0:
					job_id = current_jid
					if not is_success:
						inc_job_id = 0
				report_stat = os.stat(gc_file)
				cur.execute(siq.REPORTS_UPGRADE_LIN_SQL % (report_stat.st_ino, filename))
				files_to_del.append(filename)
			except Exception, exc:
				syslog.syslog(syslog.LOG_ERR, "Failed to upgrade report (%s). Error: %s" % (filename, str(exc)))
				raise
				sys.exit(-1)
			finally:
				if rdom:
					rdom.unlink()
				if rfile:
					rfile.close()
			
			if is_success:
				break
		
		query = "INSERT OR REPLACE INTO reports_status(policy_id, job_id, inc_job_id, last_known_good_time) VALUES('%s', %s, %d, %s)" % (pid, job_id, inc_job_id, lkg)
		cur.execute(query)
		conn.commit()
		cur.close()
		for xmlfile in files_to_del:
			os.unlink(xmlfile)
		syslog.syslog(syslog.LOG_INFO, "%d reports upgraded.  Current Job ID: %s.  Increment Job ID: %d.  LKG: %s." % (counter, job_id, inc_job_id, lkg))

def upgrade_reportdb():
	if (os.access(reports_db_path, os.F_OK)):
		#Reports db exists, so we must be 6.5+.  Upgrade it in place
		syslog.syslog(syslog.LOG_INFO, "Upgrading report database...")
		conn = sqlite.connect(reports_db_path)
		syslog.syslog(syslog.LOG_INFO, "Adding indices to reports...")
		conn.executescript(siq.REPORTS_CREATE_INDICIES_SQL)
		syslog.syslog(syslog.LOG_INFO, "Creating reports_status table...")
		conn.execute(siq.REPORTS_STATUS_CREATE_SQL)
		syslog.syslog(syslog.LOG_INFO, "Committing changes...")
		conn.commit()
	else:
		# Do full treewalk, but don't upgrade .xml files to .gc
		syslog.syslog(syslog.LOG_INFO, "Creating report database...")
		siqu.create_reportdb()
		syslog.syslog(syslog.LOG_INFO, "Treewalking reports...")
		upgrade_reports()

def reportsdb_policy_sync_lite():
	# Remove all entries from reportdb that reference non-existent policies
	syslog.syslog(syslog.LOG_INFO, "Cleaning report database...")
	siq_obj = siq.SyncIQ()
	policies = siq_obj.get_policies()
	
	conn = sqlite.connect(reports_db_path)
	query = "SELECT DISTINCT policy_id FROM reports;"
	cur = conn.execute(query)
	rows = cur.fetchall()
	for row in rows:
		if row[0] not in policies:
			syslog.syslog(syslog.LOG_INFO, "Removing %s reports from reports database" % (row[0]))
			query = siq.REPORTS_DELETE_PID_SQL % row[0]
			conn.execute(query)
	
	conn.commit()

# UPGRADE REPORTS
syslog.syslog(syslog.LOG_INFO, "Upgrading reports...")
if (os.access(reports_base_path, os.F_OK)):
	try:
		upgrade_reportdb()
		upgrade_lkg_reports()
		reportsdb_policy_sync_lite()
	except Exception, exc:
		syslog.syslog(syslog.LOG_ERR, "Failed to upgrade reports: %s" % str(exc))
		raise
		sys.exit(-1)
else:
	syslog.syslog(syslog.LOG_INFO, "Reports directory not found, skipping")

def parse_global_config():
	# Build a skeleton
	gc = {}
	gc["application"] = {}
	gc["coordinator"] = {}
	gc["coordinator"]["ports"] = {}
	gc["bwt"] = {}
	gc["scheduler"] = {}
	gc["reports"] = {}
	gc["reports"]["addresses"] = []
	
	#Open base xml and parse
	base_file = None
	base_dom = None
	try:
		if (os.access(global_config_path, os.F_OK)):
			base_file, base_dom = open_and_parse_xml(global_config_path)
			parse_global_config_base(base_dom, gc)
	except Exception, exc:
		print("Failed to upgrade global config base. Error: %s" % exc)
		#If the base did not parse, we cannot apply overrides on top of it
		return
	finally:
		if base_dom:
			base_dom.unlink()
		if base_file:
			base_file.close()
	
	#Open override xml and parse
	over_file = None
	over_dom = None
	if (os.access(global_config_override_path, os.F_OK)):
		try:
			over_file, over_dom = open_and_parse_xml(global_config_override_path)
			parse_global_config_override(over_dom, gc)
		except Exception, exc:
			syslog.syslog(syslog.LOG_ERR, "Failed to upgrade global config override. Error: %s" % exc)
			sys.exit(-1)
		finally:
			if over_dom:
				over_dom.unlink()
			if over_file:
				over_file.close()
	else:
		syslog.syslog(syslog.LOG_INFO, "Skipping override tsm.xml")
	
	#Save
	glob_cfg = siqu.get_gconfig_global_config()
	open_and_write_gc(glob_cfg, gc)
		

def parse_global_config_base(xml_root, gc):
	# Iterate base XML and populate each section
	# Application
	set_dict_int(gc["application"], "max_sworkers_per_node", xml_root, "max-sworkers-per-node")
	set_dict_int(gc["application"], "deny_fallback_min_ver_override", xml_root, "deny-fallback-min-ver-override")
	set_dict_int(gc["application"], "connect_timeout", xml_root, "connect-timeout")
	set_dict_bool(gc["application"], "force_sparse_writes", xml_root, "force-sparse-writes")
	# State
	gc["state"] = get_xml_data(xml_root, "state")
	# Coordinator
	# Coordinator>Ports
	set_dict_int(gc["coordinator"]["ports"], "pworker", xml_root, "pworker", True)
	set_dict_int(gc["coordinator"]["ports"], "sworker", xml_root, "sworker", True)
	set_dict_int(gc["coordinator"], "workers_per_node", xml_root, "workers-per-node")
	# Coordinator>Paths
	gc["coordinator"]["paths"] = parse_incexc(xml_root)
	gc["coordinator"]["passwd_file"] = get_xml_data(xml_root, "passwd-file")
	gc["coordinator"]["snapshot_pattern"] = get_xml_data(xml_root, "snapshot-pattern")
	set_dict_long(gc["coordinator"], "pause_limit", xml_root, "pause-limit")
	set_dict_int(gc["coordinator"], "samba_backup_enabled", xml_root, "samba-backup-enabled")
	set_dict_int(gc["coordinator"], "max_wperpolicy", xml_root, "max-wperpolicy")
	set_dict_int(gc["coordinator"], "wsock_buf_size", xml_root, "wsock-buf-size")
	set_dict_int(gc["coordinator"], "wread_buf_size", xml_root, "wread-buf-size")
	# From policies file earlier
	if restrict_by:
		gc["coordinator"]["default_restrict_by"] = restrict_by
	if target_restrict != None:
		gc["coordinator"]["default_target_restrict"] = target_restrict
	if force_interface != None:
		gc["coordinator"]["default_force_interface"] = force_interface
	# BWT
	gc["bwt"]["host"] = get_xml_data(xml_root, "host")
	set_dict_int(gc["bwt"], "port", xml_root, "port", True)
	set_dict_int(gc["bwt"], "cpu_port", xml_root, "cpu-port", True)
	gc["bwt"]["bw_key_file"] = get_xml_data(xml_root, "bw-key-file")
	gc["bwt"]["th_key_file"] = get_xml_data(xml_root, "th-key-file")
	gc["bwt"]["cpu_key_file"] = get_xml_data(xml_root, "cpu-key-file")
	set_dict_int(gc["bwt"], "log_rotate_period", xml_root, "log-rotate-period")
	# Scheduler
	gc["scheduler"]["pid_file"] = get_xml_data(xml_root, "pid-file")
	set_dict_long(gc["scheduler"], "quorum_poll_interval", xml_root, "quorum-poll-interval")
	set_dict_long(gc["scheduler"], "policies_poll_interval", xml_root, "policies-poll-interval")
	set_dict_int(gc["scheduler"], "max_concurrent_jobs", xml_root, "max-concurrent-jobs")
	# Reports
	gc["reports"]["rotation_period"] = get_xml_data(xml_root, "rotation-period")
	addrs_xnode = get_xml_node(xml_root, "addresses")
	gc["reports"]["addresses"] = parse_stringlist(addrs_xnode, "address")
	set_dict_bool(gc["reports"], "auto_send_email", xml_root, "auto-send-email")
	set_dict_int(gc["reports"], "max", xml_root, "max-reports")
	
	return gc

gc_map = {
	# Format: <MXD isi-meta-tag id> => (<gconfig>, [<categories>...])
	"max-sworkers-per-node": ["application", "max_sworkers_per_node"],
	"force-sparse-writes": ["application", "force_sparse_writes"],
	"state": ["state"],
	"ports-pworker": ["coordinator", "ports", "pworker"],
	"ports-sworker": ["coordinator", "ports", "sworker"],
	"perf-workers-per-node": ["coordinator", "workers_per_node"],
	"passwd-file": ["coordinator", "passwd_file"],
	"snapshot-pattern": ["coordinator", "snapshot_pattern"],
	"pause-limit": ["coordinator", "pause_limit"],
	"samba-backup-enabled": ["coordinator", "samba_backup_enabled"],
	"max-workers-per-policy": ["coordinator", "max_wperpolicy"],
	"wsock-buf-size": ["coordinator", "wsock_buf_size"],
	"wread-buf-size": ["coordinator", "rsock_buf_size"],
	"bwt-host": ["bwt", "host"],
	"bwt-port": ["bwt", "port"],
	"cpu-port": ["bwt", "cpu_port"],
	"bwt-bw-key-file": ["bwt", "bw_key_file"],
	"bwt-th-key-file": ["bwt", "th_key_file"],
	"bwt-cpu-key-file": ["bwt", "cpu_key_file"],
	"sched-log-rotate-period": ["bwt", "log_rotate_period"],
	"quorum-poll-interval": ["scheduler", "quorum_poll_interval"],
	"policies-poll-interval": ["scheduler", "policies_poll_interval"],
	"max-concurrent-jobs": ["scheduler", "max_concurrent_jobs"],
	"reports-rotation-period": ["reports", "rotation_period"],
	"reports-auto": ["reports", "auto_send_email"],
	"reports-max": ["reports", "max"]
}

def recursive_set_gc_value(gc_tree, value, hash_chain, count):
	if (len(hash_chain) == 1):
		gc_tree[hash_chain[0]] = value
		return gc_tree
	if count < (len(hash_chain) - 1):
		returned = recursive_set_gc_value(gc_tree[hash_chain[count]], value, hash_chain, count+1)
		gc_tree[hash_chain[count]] = returned
		return gc_tree
	else:
		gc_tree[hash_chain[count]] = value
		return gc_tree
		

def parse_global_config_override(xml_root, gc):
	# Iterate override XML and overwrite
	# Handle all the modify-text tags
	nodes = xml_root.getElementsByTagName("modify-text")
	for node in nodes:
		meta_id = str(node.attributes["id"].value)
		if gc_map.has_key(meta_id):
			if len(node.childNodes) > 0:
				value = str(node.childNodes[0].data)
				try:
					value = int(value)
				except:
					pass
				if (value == "true"):
					value = 1
				if (value == "false"):
					value = 0
				gc = recursive_set_gc_value(gc, value, gc_map[meta_id], 0)
	# Handle add-tag (only for report-addresses)
	nodes = xml_root.getElementsByTagName("add-tag")
	for node in nodes:
		meta_id = str(node.attributes["id"].value)
		if meta_id == "reports-addresses":
			addr_node = node.getElementsByTagName("addr")
			addr = str(addr_node[0].childNodes[0].data)
			gc["reports"]["addresses"].append({"msg": addr})

# UPGRADE GLOBAL CONFIGURATION
syslog.syslog(syslog.LOG_INFO, "Upgrading global config...")
parse_global_config()

def parseDays(str):
	"""
	The same as in SyncIQ BandRule class
	From external to internal representation
	"""
	alpha = 'XMTWRFS-,;'
	table = ['1111111---', '-------204', '3333333---', '--------04']
	res = state = day = 0
	for c in str + ';':
		save_c = c
		c = c.upper()
		if state == 4:
			break
		if not c in alpha or table[state][alpha.find(c)] == '-':
			syslog.syslog(syslog.LOG_ERR, "parseDays: Invalid day specified (%s)" % save_c)
			raise Exception("Invalid day specified: %s" % save_c)
		if state == 0:
			day = alpha.find(c)
		elif state == 1 and c in ',;':
			res |= 1 << day
		elif state == 2:
			for i in range(day, alpha.find(c) + 1):
				res |= 1 << i
		state = int(table[state][alpha.find(c)])
	if not res:
		syslog.syslog(syslog.LOG_ERR, "The days interval cannot be empty")
		raise Exception("The days interval cannot be empty")    
	return res

def parseTimes(str):
	"""From external to internal representation"""
	times = re.match('^(\d*):(\d*)-(\d*):(\d*)$', str)
	try:
		(s, e) = (int(times.group(1)) * 60 + int(times.group(2)),
		int(times.group(3)) * 60 + int(times.group(4)))
		if e < s or e >= 1440 or s >= 1440:
			raise PyError("")
	except Exception, exc:
		syslog.syslog(syslog.LOG_ERR, "parseTimes: Invalid time")
		raise exc
	return (s, e)
		
# UPGRADE BWT
def parse_bwt(xml_root):
	to_return = {"bw": [], "cpu": [], "th": []}
	add_nodes = xml_root.getElementsByTagName("add-tag")
	for add in add_nodes:
		bwt_type = str(add.attributes["id"].value)
		limit = get_xml_node(add, "limit")
		lim_gc = {"schedule": {}}
		days_str = parseDays(get_xml_data(limit, "days"))
		if (days_str != None):
			lim_gc["schedule"]["days"] = str(days_str)
		start_str = get_xml_data(limit, "start")
		end_str = get_xml_data(limit, "end")
		(start, end) = parseTimes(start_str + "-" + end_str)
		lim_gc["schedule"]["start"] = start
		lim_gc["schedule"]["end"] = end
		lim_gc["desc"] = get_xml_data(limit, "desc")
		lim_gc["limit"] = get_field_long(limit, "value")
		lim_gc["state"] = get_xml_data(limit, "state")
		if (bwt_type == "bw-limit"):
			to_return["bw"].append(lim_gc)
		elif (bwt_type == "th-limit"):
			to_return["th"].append(lim_gc)
		elif (bwt_type == "cpu-limit"):
			to_return["cpu"].append(lim_gc)
		else:
			syslog.syslog(syslog.LOG_ERR, "Invalid bwt_type (%s)" % bwt_type)
	return to_return

bwt_file = None
bwt_dom = None
syslog.syslog(syslog.LOG_INFO, "Upgrading BWT...")

if (os.access(bwt_path, os.F_OK)):
	try:
		bwt_file, bwt_dom = open_and_parse_xml(bwt_path)
		bwt_gc = parse_bwt(bwt_dom)
		bcfg = siqu.get_gconfig_bwt()
		open_and_write_gc(bcfg, bwt_gc)
	except Exception, exc:
		syslog.syslog(syslog.LOG_ERR, "Failed to upgrade BWT. Error: %s" % exc)
		sys.exit(-1)
	finally:
		if bwt_dom:
			bwt_dom.unlink()
		if bwt_file:
			bwt_file.close()
else:
	syslog.syslog(syslog.LOG_INFO, "Skipping BWT")

# Clean up
# Remove existing xml files
rmxmlcmd = "find %s -name '*.xml' -exec rm -rf {} \;" % (path_to_backup)
#TODO os.system(rmxmlcmd)

# END OF UPGRADE - Set status for exiting function
exit_status = 0

