source/code/troubleshooter/modules/log_collector/update_mgmt_health_check.py [22:401]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - rule_info_list = [] output = [] oms_admin_conf_path = "/etc/opt/microsoft/omsagent/conf/omsadmin.conf" oms_agent_dir = "/var/opt/microsoft/omsagent" oms_agent_log = "/var/opt/microsoft/omsagent/log/omsagent.log" current_mof = "/etc/opt/omi/conf/omsconfig/configuration/Current.mof" status_passed = "Passed" status_failed = "Failed" status_debug = "Debug" empty_failure_reason = "" workspace = "" class RuleInfo: def __init__(self, rule_id, rule_group_id, status, result_msg_id): self.RuleId = rule_id self.RuleGroupId = rule_group_id self.CheckResult = status self.CheckResultMessageId = result_msg_id self.CheckResultMessageArguments = list() def main(output_path=None, return_json_output="False"): if os.geteuid() != 0: print("Please run this script as root") exit() # supported python version 2.4.x to 2.7.x if not ((sys.version_info[0] == 2) and ((sys.version_info[1]>=4) and (sys.version_info[1] < 8))): log_msg = "Unsupport python version:%s.%s. Supported python version (2.4 to 2.7)" % (sys.version_info[0], sys.version_info[1]) print(log_msg) exit() global workspace workspace = get_workspace() get_machine_info() check_os_version() check_oms_agent_installed() check_oms_agent_running() check_multihoming() check_hybrid_worker_package_present() check_hybrid_worker_running() check_general_internet_connectivity() check_agent_service_endpoint() check_jrds_endpoint(workspace) check_log_analytics_endpoints() if return_json_output == "True": print(json.dumps([obj.__dict__ for obj in rule_info_list])) else: for line in output: print(line) if output_path is not None: try: os.makedirs(output_path) except OSError: if not os.path.isdir(output_path): raise log_path = "%s/healthcheck-%s.log" % (output_path, datetime.datetime.utcnow().isoformat()) f = open(log_path, "w") f.write("".join(output)) f.close() print("Output is written to " + log_path) def get_machine_info(): FNULL = open(os.devnull, "w") if subprocess.call(["which", "hostnamectl"], stdout=FNULL, stderr=FNULL) == 0: hostname_output = os.popen("hostnamectl").read() write_log_output(None, None, status_debug, empty_failure_reason, "Machine Information:" + hostname_output) FNULL.close() def check_os_version(): rule_id = "Linux.OperatingSystemCheck" rule_group_id = "prerequisites" os_version = platform.platform() supported_os_url = "https://docs.microsoft.com/en-us/azure/automation/automation-update-management#clients" # We support (Ubuntu 14.04, Ubuntu 16.04, Ubuntu 18.04, SuSE 11, SuSE 12, Redhat 6, Redhat 7, CentOs 6, CentOs 7) if re.search("Ubuntu-14.04", os_version, re.IGNORECASE) or \ re.search("Ubuntu-16.04", os_version, re.IGNORECASE) or \ re.search("Ubuntu-18.04", os_version, re.IGNORECASE) or \ re.search("SuSE-11", os_version, re.IGNORECASE) or \ re.search("SuSE-12", os_version, re.IGNORECASE) or \ re.search("redhat-6", os_version, re.IGNORECASE) or \ re.search("redhat-7", os_version, re.IGNORECASE) or \ re.search("centos-6", os_version, re.IGNORECASE) or \ re.search("centos-7", os_version, re.IGNORECASE) : write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Operating system version is supported") else: log_msg = "Operating System version (%s) is not supported. Supported versions listed here: %s" % (os_version, supported_os_url) write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, log_msg, supported_os_url) def check_oms_agent_installed(): rule_id = "Linux.OMSAgentInstallCheck" rule_group_id = "servicehealth" oms_agent_troubleshooting_url = "https://github.com/Microsoft/OMS-Agent-for-Linux/blob/master/docs/Troubleshooting.md" if os.path.isfile(oms_admin_conf_path) and os.path.isfile(oms_agent_log): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Microsoft Monitoring agent is installed") oms_admin_file_content = "\t" oms_admin_file = open(oms_admin_conf_path, "r") for line in oms_admin_file: oms_admin_file_content += line + "\t" oms_admin_file.close() write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "omsadmin.conf file contents:\n" + oms_admin_file_content) else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Microsoft Monitoring agent is not installed", oms_agent_troubleshooting_url) write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "Microsoft Monitoring agent troubleshooting guide:" + oms_agent_troubleshooting_url) return def check_oms_agent_running(): rule_id = "Linux.OMSAgentStatusCheck" rule_group_id = "servicehealth" oms_agent_troubleshooting_url = "https://github.com/Microsoft/OMS-Agent-for-Linux/blob/master/docs/Troubleshooting.md" is_oms_agent_running, ps_output = is_process_running("omsagent", ["omsagent.log", "omsagent.conf"], "OMS Agent") if is_oms_agent_running: write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Microsoft Monitoring agent is running") else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Microsoft Monitoring agent is not running", oms_agent_troubleshooting_url) write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, ps_output) write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "Microsoft Monitoring agent troubleshooting guide:" + oms_agent_troubleshooting_url) def check_multihoming(): rule_id = "Linux.MultiHomingCheck" rule_group_id = "servicehealth" if not os.path.isdir(oms_agent_dir): write_log_output(rule_id, rule_group_id, status_failed, "NoWorkspace", "Machine is not registered with log analytics workspace.") return directories = [] potential_workspaces = [] for (dirpath, dirnames, filenames) in walk(oms_agent_dir): directories.extend(dirnames) break # Get the top level of directories for directory in directories: if len(directory) >= 32: potential_workspaces.append(directory) workspace_id_list = str(potential_workspaces) if len(potential_workspaces) > 1: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Machine registered with more than one log analytics workspace. List of workspaces:" + workspace_id_list, workspace_id_list) else: write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Machine registered with log analytics workspace:" + workspace_id_list, workspace_id_list) def check_hybrid_worker_package_present(): rule_id = "Linux.HybridWorkerPackgeCheck" rule_group_id = "servicehealth" if os.path.isfile("/opt/microsoft/omsconfig/modules/nxOMSAutomationWorker/VERSION") and \ os.path.isfile("/opt/microsoft/omsconfig/modules/nxOMSAutomationWorker/DSCResources/MSFT_nxOMSAutomationWorkerResource/automationworker/worker/configuration.py"): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Hybrid worker package is present") else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Hybrid worker package is not present") def check_hybrid_worker_running(): rule_id = "Linux.HybridWorkerStatusCheck" rule_group_id = "servicehealth" if not os.path.isfile(current_mof): write_log_output(rule_id, rule_group_id, status_failed, "MissingCurrentMofFile", "Hybrid worker is not running. current_mof file:(" + current_mof + ") is missing", current_mof) return search_text = "ResourceSettings" command = "file -b --mime-encoding " + current_mof current_mof_encoding = os.popen(command).read() resourceSetting = find_line_in_file("ResourceSettings", current_mof, current_mof_encoding); if resourceSetting is None: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Hybrid worker is not running") write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "Unable to get ResourceSettings from current_mof file:(" + current_mof + ") with file encoding:" + current_mof_encoding) return backslash = string.replace("\str", "str", "") resourceSetting = string.replace(resourceSetting, backslash, "") resourceSetting = string.replace(resourceSetting, ";", "") resourceSetting = string.replace(resourceSetting, "\"[", "[") resourceSetting = string.replace(resourceSetting, "]\"", "]") resourceSetting = resourceSetting.split("=")[1].strip() automation_worker_path = "/opt/microsoft/omsconfig/Scripts/" if (sys.version_info.major == 2) : if (sys.version_info.minor >= 6) : automation_worker_path += "2.6x-2.7x" else: automation_worker_path += "2.4x-2.5x" os.chdir(automation_worker_path) nxOMSAutomationWorker=imp.load_source("nxOMSAutomationWorker", "./Scripts/nxOMSAutomationWorker.py") settings = nxOMSAutomationWorker.read_settings_from_mof_json(resourceSetting) if not settings.auto_register_enabled: write_log_output(rule_id, rule_group_id, status_failed, "UpdateDeploymentDisabled", "Hybrid worker is not running", current_mof) write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "Update deployment solution is not enabled. ResourceSettings:" + resourceSetting) return if nxOMSAutomationWorker.Test_Marshall(resourceSetting) == [0]: write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Hybrid worker is running") else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Hybrid worker is not running") write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "ResourceSettings:" + resourceSetting + " read from current_mof file:(" + current_mof + ")") write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "nxOMSAutomationWorker.py path:" + automation_worker_path) def check_general_internet_connectivity(): rule_id = "Linux.InternetConnectionCheck" rule_group_id = "connectivity" if check_endpoint(None, "bing.com") and check_endpoint(None, "google.com"): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Machine is connected to internet") else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Machine is not connected to internet") def check_agent_service_endpoint(): rule_id = "Linux.AgentServiceConnectivityCheck" rule_group_id = "connectivity" agent_endpoint = get_agent_endpoint() if agent_endpoint is None: write_log_output(rule_id, rule_group_id, status_failed, "UnableToGetEndpoint", "Unable to get the registration (agent service) endpoint") elif check_endpoint(None, agent_endpoint): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "TCP test for {" + agent_endpoint + "} (port 443) succeeded", agent_endpoint) else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "TCP test for {" + agent_endpoint + "} (port 443) failed", agent_endpoint) def check_jrds_endpoint(workspace): rule_id = "Linux.JRDSConnectivityCheck" rule_group_id = "connectivity" jrds_endpoint = get_jrds_endpoint(workspace) if jrds_endpoint is None: write_log_output(rule_id, rule_group_id, status_failed, "UnableToGetEndpoint", "Unable to get the operations (JRDS) endpoint") elif jrds_endpoint is not None and check_endpoint(workspace, jrds_endpoint): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "TCP test for {" + jrds_endpoint + "} (port 443) succeeded", jrds_endpoint) else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "TCP test for {" + jrds_endpoint + "} (port 443) failed", jrds_endpoint) def check_log_analytics_endpoints(): rule_id = "Linux.LogAnalyticsConnectivityCheck" rule_group_id = "connectivity" i = 0 if is_fairfax_region() is True: fairfax_log_analytics_endpoints = ["usge-jobruntimedata-prod-1.usgovtrafficmanager.net", "usge-agentservice-prod-1.usgovtrafficmanager.net", "*.ods.opinsights.azure.us", "*.oms.opinsights.azure.us" ] for endpoint in fairfax_log_analytics_endpoints: i += 1 if "*" in endpoint and workspace is not None: endpoint = endpoint.replace("*", workspace) if check_endpoint(workspace, endpoint): write_log_output(rule_id + str(i), rule_group_id, status_passed, empty_failure_reason, "TCP test for {" + endpoint + "} (port 443) succeeded", endpoint) else: write_log_output(rule_id + str(i), rule_group_id, status_failed, empty_failure_reason, "TCP test for {" + endpoint + "} (port 443) failed", endpoint) else: log_analytics_endpoints = ["*.ods.opinsights.azure.com", "*.oms.opinsights.azure.com", "ods.systemcenteradvisor.com"] for endpoint in log_analytics_endpoints: i += 1 if "*" in endpoint and workspace is not None: endpoint = endpoint.replace("*", workspace) if check_endpoint(workspace, endpoint): write_log_output(rule_id + str(i), rule_group_id, status_passed, empty_failure_reason, "TCP test for {" + endpoint + "} (port 443) succeeded", endpoint) else: write_log_output(rule_id + str(i), rule_group_id, status_failed, empty_failure_reason, "TCP test for {" + endpoint + "} (port 443) failed", endpoint) def check_endpoint(workspace, endpoint): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) new_endpoint = None if "*" in endpoint and workspace is not None: new_endpoint = endpoint.replace("*", workspace) elif "*" not in endpoint: new_endpoint = endpoint if new_endpoint is not None: try: response = sock.connect_ex((new_endpoint, 443)) if response == 0: return True else: return False except Exception as ex: return False else: return False def get_jrds_endpoint(workspace): if workspace is not None: worker_conf_path = "/var/opt/microsoft/omsagent/%s/state/automationworker/worker.conf" % (workspace) line = find_line_in_file("jrds_base_uri", worker_conf_path) if line is not None: return line.split("=")[1].split("/")[2].strip() return None def get_agent_endpoint(): line = find_line_in_file("agentservice", oms_admin_conf_path) # try checking under different name if line is None: line = find_line_in_file("DSC_ENDPOINT", oms_admin_conf_path) # Fetch the text after https:// if line is not None: return line.split("=")[1].split("/")[2].strip() return None def is_process_running(process_name, search_criteria, output_name): command = "ps aux | grep %s | grep -v grep" % (process_name) grep_output = os.popen(command).read() if any(search_text in grep_output for search_text in search_criteria): return True, grep_output else: return False, grep_output def get_workspace(): line = find_line_in_file("WORKSPACE", oms_admin_conf_path) if line is not None: return line.split("=")[1].strip() return None def is_fairfax_region(): oms_endpoint = find_line_in_file("OMS_ENDPOINT", oms_admin_conf_path) if oms_endpoint is not None: return ".us" in oms_endpoint.split("=")[1] def find_line_in_file(search_text, path, file_encoding=""): if os.path.isfile(path): if file_encoding == "": current_file = open(path, "r") else: current_file = codecs.open(path, "r", file_encoding) for line in current_file: if search_text in line: current_file.close() return line current_file.close() return None def write_log_output(rule_id, rule_group_id, status, failure_reason, log_msg, *result_msg_args): global output, rule_info_list if(type(log_msg) != str): log_msg = str(log_msg) if status != status_debug: if failure_reason == empty_failure_reason: result_msg_id = rule_id + "." + status else: result_msg_id = rule_id + "." + status + "." + failure_reason current_rule_info = RuleInfo(rule_id, rule_group_id, status, result_msg_id) result_msg_args_list = [] for arg in result_msg_args: current_rule_info.CheckResultMessageArguments.append(arg) rule_info_list.append(current_rule_info) output.append(status + ": " + log_msg + "\n") if __name__ == "__main__": if len(sys.argv) > 2: main(sys.argv[1], sys.argv[2]) elif len(sys.argv) > 1: main(sys.argv[1]) else: main() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - tools/LogCollector/source/update_mgmt_health_check.py [22:401]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - rule_info_list = [] output = [] oms_admin_conf_path = "/etc/opt/microsoft/omsagent/conf/omsadmin.conf" oms_agent_dir = "/var/opt/microsoft/omsagent" oms_agent_log = "/var/opt/microsoft/omsagent/log/omsagent.log" current_mof = "/etc/opt/omi/conf/omsconfig/configuration/Current.mof" status_passed = "Passed" status_failed = "Failed" status_debug = "Debug" empty_failure_reason = "" workspace = "" class RuleInfo: def __init__(self, rule_id, rule_group_id, status, result_msg_id): self.RuleId = rule_id self.RuleGroupId = rule_group_id self.CheckResult = status self.CheckResultMessageId = result_msg_id self.CheckResultMessageArguments = list() def main(output_path=None, return_json_output="False"): if os.geteuid() != 0: print("Please run this script as root") exit() # supported python version 2.4.x to 2.7.x if not ((sys.version_info[0] == 2) and ((sys.version_info[1]>=4) and (sys.version_info[1] < 8))): log_msg = "Unsupport python version:%s.%s. Supported python version (2.4 to 2.7)" % (sys.version_info[0], sys.version_info[1]) print(log_msg) exit() global workspace workspace = get_workspace() get_machine_info() check_os_version() check_oms_agent_installed() check_oms_agent_running() check_multihoming() check_hybrid_worker_package_present() check_hybrid_worker_running() check_general_internet_connectivity() check_agent_service_endpoint() check_jrds_endpoint(workspace) check_log_analytics_endpoints() if return_json_output == "True": print(json.dumps([obj.__dict__ for obj in rule_info_list])) else: for line in output: print(line) if output_path is not None: try: os.makedirs(output_path) except OSError: if not os.path.isdir(output_path): raise log_path = "%s/healthcheck-%s.log" % (output_path, datetime.datetime.utcnow().isoformat()) f = open(log_path, "w") f.write("".join(output)) f.close() print("Output is written to " + log_path) def get_machine_info(): FNULL = open(os.devnull, "w") if subprocess.call(["which", "hostnamectl"], stdout=FNULL, stderr=FNULL) == 0: hostname_output = os.popen("hostnamectl").read() write_log_output(None, None, status_debug, empty_failure_reason, "Machine Information:" + hostname_output) FNULL.close() def check_os_version(): rule_id = "Linux.OperatingSystemCheck" rule_group_id = "prerequisites" os_version = platform.platform() supported_os_url = "https://docs.microsoft.com/en-us/azure/automation/automation-update-management#clients" # We support (Ubuntu 14.04, Ubuntu 16.04, Ubuntu 18.04, SuSE 11, SuSE 12, Redhat 6, Redhat 7, CentOs 6, CentOs 7) if re.search("Ubuntu-14.04", os_version, re.IGNORECASE) or \ re.search("Ubuntu-16.04", os_version, re.IGNORECASE) or \ re.search("Ubuntu-18.04", os_version, re.IGNORECASE) or \ re.search("SuSE-11", os_version, re.IGNORECASE) or \ re.search("SuSE-12", os_version, re.IGNORECASE) or \ re.search("redhat-6", os_version, re.IGNORECASE) or \ re.search("redhat-7", os_version, re.IGNORECASE) or \ re.search("centos-6", os_version, re.IGNORECASE) or \ re.search("centos-7", os_version, re.IGNORECASE) : write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Operating system version is supported") else: log_msg = "Operating System version (%s) is not supported. Supported versions listed here: %s" % (os_version, supported_os_url) write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, log_msg, supported_os_url) def check_oms_agent_installed(): rule_id = "Linux.OMSAgentInstallCheck" rule_group_id = "servicehealth" oms_agent_troubleshooting_url = "https://github.com/Microsoft/OMS-Agent-for-Linux/blob/master/docs/Troubleshooting.md" if os.path.isfile(oms_admin_conf_path) and os.path.isfile(oms_agent_log): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Microsoft Monitoring agent is installed") oms_admin_file_content = "\t" oms_admin_file = open(oms_admin_conf_path, "r") for line in oms_admin_file: oms_admin_file_content += line + "\t" oms_admin_file.close() write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "omsadmin.conf file contents:\n" + oms_admin_file_content) else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Microsoft Monitoring agent is not installed", oms_agent_troubleshooting_url) write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "Microsoft Monitoring agent troubleshooting guide:" + oms_agent_troubleshooting_url) return def check_oms_agent_running(): rule_id = "Linux.OMSAgentStatusCheck" rule_group_id = "servicehealth" oms_agent_troubleshooting_url = "https://github.com/Microsoft/OMS-Agent-for-Linux/blob/master/docs/Troubleshooting.md" is_oms_agent_running, ps_output = is_process_running("omsagent", ["omsagent.log", "omsagent.conf"], "OMS Agent") if is_oms_agent_running: write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Microsoft Monitoring agent is running") else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Microsoft Monitoring agent is not running", oms_agent_troubleshooting_url) write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, ps_output) write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "Microsoft Monitoring agent troubleshooting guide:" + oms_agent_troubleshooting_url) def check_multihoming(): rule_id = "Linux.MultiHomingCheck" rule_group_id = "servicehealth" if not os.path.isdir(oms_agent_dir): write_log_output(rule_id, rule_group_id, status_failed, "NoWorkspace", "Machine is not registered with log analytics workspace.") return directories = [] potential_workspaces = [] for (dirpath, dirnames, filenames) in walk(oms_agent_dir): directories.extend(dirnames) break # Get the top level of directories for directory in directories: if len(directory) >= 32: potential_workspaces.append(directory) workspace_id_list = str(potential_workspaces) if len(potential_workspaces) > 1: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Machine registered with more than one log analytics workspace. List of workspaces:" + workspace_id_list, workspace_id_list) else: write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Machine registered with log analytics workspace:" + workspace_id_list, workspace_id_list) def check_hybrid_worker_package_present(): rule_id = "Linux.HybridWorkerPackgeCheck" rule_group_id = "servicehealth" if os.path.isfile("/opt/microsoft/omsconfig/modules/nxOMSAutomationWorker/VERSION") and \ os.path.isfile("/opt/microsoft/omsconfig/modules/nxOMSAutomationWorker/DSCResources/MSFT_nxOMSAutomationWorkerResource/automationworker/worker/configuration.py"): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Hybrid worker package is present") else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Hybrid worker package is not present") def check_hybrid_worker_running(): rule_id = "Linux.HybridWorkerStatusCheck" rule_group_id = "servicehealth" if not os.path.isfile(current_mof): write_log_output(rule_id, rule_group_id, status_failed, "MissingCurrentMofFile", "Hybrid worker is not running. current_mof file:(" + current_mof + ") is missing", current_mof) return search_text = "ResourceSettings" command = "file -b --mime-encoding " + current_mof current_mof_encoding = os.popen(command).read() resourceSetting = find_line_in_file("ResourceSettings", current_mof, current_mof_encoding); if resourceSetting is None: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Hybrid worker is not running") write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "Unable to get ResourceSettings from current_mof file:(" + current_mof + ") with file encoding:" + current_mof_encoding) return backslash = string.replace("\str", "str", "") resourceSetting = string.replace(resourceSetting, backslash, "") resourceSetting = string.replace(resourceSetting, ";", "") resourceSetting = string.replace(resourceSetting, "\"[", "[") resourceSetting = string.replace(resourceSetting, "]\"", "]") resourceSetting = resourceSetting.split("=")[1].strip() automation_worker_path = "/opt/microsoft/omsconfig/Scripts/" if (sys.version_info.major == 2) : if (sys.version_info.minor >= 6) : automation_worker_path += "2.6x-2.7x" else: automation_worker_path += "2.4x-2.5x" os.chdir(automation_worker_path) nxOMSAutomationWorker=imp.load_source("nxOMSAutomationWorker", "./Scripts/nxOMSAutomationWorker.py") settings = nxOMSAutomationWorker.read_settings_from_mof_json(resourceSetting) if not settings.auto_register_enabled: write_log_output(rule_id, rule_group_id, status_failed, "UpdateDeploymentDisabled", "Hybrid worker is not running", current_mof) write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "Update deployment solution is not enabled. ResourceSettings:" + resourceSetting) return if nxOMSAutomationWorker.Test_Marshall(resourceSetting) == [0]: write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Hybrid worker is running") else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Hybrid worker is not running") write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "ResourceSettings:" + resourceSetting + " read from current_mof file:(" + current_mof + ")") write_log_output(rule_id, rule_group_id, status_debug, empty_failure_reason, "nxOMSAutomationWorker.py path:" + automation_worker_path) def check_general_internet_connectivity(): rule_id = "Linux.InternetConnectionCheck" rule_group_id = "connectivity" if check_endpoint(None, "bing.com") and check_endpoint(None, "google.com"): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "Machine is connected to internet") else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "Machine is not connected to internet") def check_agent_service_endpoint(): rule_id = "Linux.AgentServiceConnectivityCheck" rule_group_id = "connectivity" agent_endpoint = get_agent_endpoint() if agent_endpoint is None: write_log_output(rule_id, rule_group_id, status_failed, "UnableToGetEndpoint", "Unable to get the registration (agent service) endpoint") elif check_endpoint(None, agent_endpoint): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "TCP test for {" + agent_endpoint + "} (port 443) succeeded", agent_endpoint) else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "TCP test for {" + agent_endpoint + "} (port 443) failed", agent_endpoint) def check_jrds_endpoint(workspace): rule_id = "Linux.JRDSConnectivityCheck" rule_group_id = "connectivity" jrds_endpoint = get_jrds_endpoint(workspace) if jrds_endpoint is None: write_log_output(rule_id, rule_group_id, status_failed, "UnableToGetEndpoint", "Unable to get the operations (JRDS) endpoint") elif jrds_endpoint is not None and check_endpoint(workspace, jrds_endpoint): write_log_output(rule_id, rule_group_id, status_passed, empty_failure_reason, "TCP test for {" + jrds_endpoint + "} (port 443) succeeded", jrds_endpoint) else: write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason, "TCP test for {" + jrds_endpoint + "} (port 443) failed", jrds_endpoint) def check_log_analytics_endpoints(): rule_id = "Linux.LogAnalyticsConnectivityCheck" rule_group_id = "connectivity" i = 0 if is_fairfax_region() is True: fairfax_log_analytics_endpoints = ["usge-jobruntimedata-prod-1.usgovtrafficmanager.net", "usge-agentservice-prod-1.usgovtrafficmanager.net", "*.ods.opinsights.azure.us", "*.oms.opinsights.azure.us" ] for endpoint in fairfax_log_analytics_endpoints: i += 1 if "*" in endpoint and workspace is not None: endpoint = endpoint.replace("*", workspace) if check_endpoint(workspace, endpoint): write_log_output(rule_id + str(i), rule_group_id, status_passed, empty_failure_reason, "TCP test for {" + endpoint + "} (port 443) succeeded", endpoint) else: write_log_output(rule_id + str(i), rule_group_id, status_failed, empty_failure_reason, "TCP test for {" + endpoint + "} (port 443) failed", endpoint) else: log_analytics_endpoints = ["*.ods.opinsights.azure.com", "*.oms.opinsights.azure.com", "ods.systemcenteradvisor.com"] for endpoint in log_analytics_endpoints: i += 1 if "*" in endpoint and workspace is not None: endpoint = endpoint.replace("*", workspace) if check_endpoint(workspace, endpoint): write_log_output(rule_id + str(i), rule_group_id, status_passed, empty_failure_reason, "TCP test for {" + endpoint + "} (port 443) succeeded", endpoint) else: write_log_output(rule_id + str(i), rule_group_id, status_failed, empty_failure_reason, "TCP test for {" + endpoint + "} (port 443) failed", endpoint) def check_endpoint(workspace, endpoint): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) new_endpoint = None if "*" in endpoint and workspace is not None: new_endpoint = endpoint.replace("*", workspace) elif "*" not in endpoint: new_endpoint = endpoint if new_endpoint is not None: try: response = sock.connect_ex((new_endpoint, 443)) if response == 0: return True else: return False except Exception as ex: return False else: return False def get_jrds_endpoint(workspace): if workspace is not None: worker_conf_path = "/var/opt/microsoft/omsagent/%s/state/automationworker/worker.conf" % (workspace) line = find_line_in_file("jrds_base_uri", worker_conf_path) if line is not None: return line.split("=")[1].split("/")[2].strip() return None def get_agent_endpoint(): line = find_line_in_file("agentservice", oms_admin_conf_path) # try checking under different name if line is None: line = find_line_in_file("DSC_ENDPOINT", oms_admin_conf_path) # Fetch the text after https:// if line is not None: return line.split("=")[1].split("/")[2].strip() return None def is_process_running(process_name, search_criteria, output_name): command = "ps aux | grep %s | grep -v grep" % (process_name) grep_output = os.popen(command).read() if any(search_text in grep_output for search_text in search_criteria): return True, grep_output else: return False, grep_output def get_workspace(): line = find_line_in_file("WORKSPACE", oms_admin_conf_path) if line is not None: return line.split("=")[1].strip() return None def is_fairfax_region(): oms_endpoint = find_line_in_file("OMS_ENDPOINT", oms_admin_conf_path) if oms_endpoint is not None: return ".us" in oms_endpoint.split("=")[1] def find_line_in_file(search_text, path, file_encoding=""): if os.path.isfile(path): if file_encoding == "": current_file = open(path, "r") else: current_file = codecs.open(path, "r", file_encoding) for line in current_file: if search_text in line: current_file.close() return line current_file.close() return None def write_log_output(rule_id, rule_group_id, status, failure_reason, log_msg, *result_msg_args): global output, rule_info_list if(type(log_msg) != str): log_msg = str(log_msg) if status != status_debug: if failure_reason == empty_failure_reason: result_msg_id = rule_id + "." + status else: result_msg_id = rule_id + "." + status + "." + failure_reason current_rule_info = RuleInfo(rule_id, rule_group_id, status, result_msg_id) result_msg_args_list = [] for arg in result_msg_args: current_rule_info.CheckResultMessageArguments.append(arg) rule_info_list.append(current_rule_info) output.append(status + ": " + log_msg + "\n") if __name__ == "__main__": if len(sys.argv) > 2: main(sys.argv[1], sys.argv[2]) elif len(sys.argv) > 1: main(sys.argv[1]) else: main() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -