From 1e57d3059736af24185c9616c4e5ed49634ca97d Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Tue, 2 Jul 2024 20:42:34 +0100 Subject: [PATCH 1/3] Some changes done to log CPU/RAM to MS SQL. More changes to be done to log URL monitors to MS SQL too, along with making a compatible MySQL/MariaDB equiv... --- log.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/log.py b/log.py index e69de29..925af71 100644 --- a/log.py +++ b/log.py @@ -0,0 +1,27 @@ +import pyodbc +from datetime import datetime + +class logsManager: + + def __init__(self, server, database, username, password): + self.conn_str = 'DRIVER={ODBC Driver 17 for SQL Server};SERVER=' + server + ';DATABASE=' + database + ';UID=' + username + ';PWD=' + password + + def insert_host_log(self, hostname, log_time, cpu, memory): + try: + conn = pyodbc.connect(self.conn_str) + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) FROM monutil_hosts WHERE hostname = ?", hostname) + + if cursor.fetchone()[0] == 0: + + cursor.execute("INSERT INTO monutil_hosts (hostname, ipAddress) VALUES (?, '127.0.0.1')", hostname) + + cursor.execute("INSERT INTO monutil_hostlogs (hostname, logTime, cpu, memory) VALUES (?, ?, ?, ?)", hostname, log_time, cpu, memory) + conn.commit() + + conn.close() + + except pyodbc.Error as ex: + + print("Error inserting data: {}".format(str(ex))) From 6644051c7b911ea3bd323f7cdb0ef7feac90bff9 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Tue, 2 Jul 2024 20:46:36 +0100 Subject: [PATCH 2/3] Correction to last commit --- main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/main.py b/main.py index 1e6b5e0..6318a92 100755 --- a/main.py +++ b/main.py @@ -30,9 +30,12 @@ import log import requests import threading import signal +import socket from functools import partial from concurrent.futures import ThreadPoolExecutor, as_completed from bs4 import BeautifulSoup +from log import hostLogsManager +from datetime import datetime stop_event = threading.Event() @@ -61,6 +64,7 @@ def monitorHost(stop_event): loadavg = round((load1/os.cpu_count()) * 100, 2) memory = psutil.virtual_memory().percent + logHostLog(socket.gethostname(), datetime.now(), loadavg, memory) print("CPU %: " + str(loadavg)) print("Memory %: " + str(memory)) @@ -104,6 +108,11 @@ def monitorUrls(stop_event): time.sleep(config.urlMonitoringPeriod) +def logHostLog(hostname, logTime, cpu, memory): + + manager = hostLogsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) + manager.insert_host_log(hostname, logTime, cpu, memory) + def main(): signal.signal(signal.SIGTERM, signal_handler) From 5bf9beb1bc03d62302394eac020947faf5a7d016 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Wed, 3 Jul 2024 21:23:36 +0100 Subject: [PATCH 3/3] Now submits URL logs and hostlogs --- config.py | 5 +++++ log.py | 52 ++++++++++++++++++++++++++++++++++++++++++++-------- main.py | 13 ++++++++++--- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/config.py b/config.py index 3fc66e9..18c5922 100644 --- a/config.py +++ b/config.py @@ -5,3 +5,8 @@ urlMonitoringPeriod = 30 urls = ["https://www.bootlesshacker.com"] urlTimeout = 10 maxWorkers = 4 +forceNonPOSIXCPU = False # this will force the custom cpu monintor to run instead if set to True +sqlServer = '' +sqlDatabase = '' +sqlUsername = '' +sqlPassword = '' diff --git a/log.py b/log.py index 925af71..2f8af2b 100644 --- a/log.py +++ b/log.py @@ -6,22 +6,58 @@ class logsManager: def __init__(self, server, database, username, password): self.conn_str = 'DRIVER={ODBC Driver 17 for SQL Server};SERVER=' + server + ';DATABASE=' + database + ';UID=' + username + ';PWD=' + password - def insert_host_log(self, hostname, log_time, cpu, memory): + def insertHost(self, hostname, ipAddress): + try: + conn = pyodbc.connect(self.conn_str) cursor = conn.cursor() - cursor.execute("SELECT COUNT(*) FROM monutil_hosts WHERE hostname = ?", hostname) if cursor.fetchone()[0] == 0: - cursor.execute("INSERT INTO monutil_hosts (hostname, ipAddress) VALUES (?, '127.0.0.1')", hostname) + cursor.execute("INSERT INTO monutil_hosts (hostname, ipAddress) VALUES (?, ?)", hostname, ipAddress) + conn.commit() + + else: + + cursor.execute("UPDATE monutil_hosts SET ipAddress = ? WHERE hostname = ?", ipAddress, hostname) - cursor.execute("INSERT INTO monutil_hostlogs (hostname, logTime, cpu, memory) VALUES (?, ?, ?, ?)", hostname, log_time, cpu, memory) - conn.commit() - - conn.close() - except pyodbc.Error as ex: print("Error inserting data: {}".format(str(ex))) + + def insertHostLog(self, hostname, ipAddress, log_time, cpu, memory): + + try: + + self.insertHost(hostname, ipAddress) + + conn = pyodbc.connect(self.conn_str) + cursor = conn.cursor() + + cursor.execute("INSERT INTO monutil_hostlogs (hostname, logTime, cpu, memory) VALUES (?, ?, ?, ?)", hostname, log_time, cpu, memory) + conn.commit() + conn.close() + + except pyodbc.Error as ex: + + print("Error inserting data: {}".format(str(ex))) + + def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime): + + try: + + self.insertHost(hostname, ipAddress) + + conn = pyodbc.connect(self.conn_str) + cursor = conn.cursor() + + cursor.execute("INSERT INTO monutil_urlLogs (hostname, url, logTime, responseTime) VALUES (?, ?, ?, ?)", hostname, url, log_time, responseTime) + + conn.commit() + conn.close() + + except pyodbc.Error as ex: + + print("Error inserting data into monutil_urlLogs:", ex) diff --git a/main.py b/main.py index 6318a92..342b266 100755 --- a/main.py +++ b/main.py @@ -34,7 +34,7 @@ import socket from functools import partial from concurrent.futures import ThreadPoolExecutor, as_completed from bs4 import BeautifulSoup -from log import hostLogsManager +from log import logsManager from datetime import datetime stop_event = threading.Event() @@ -102,6 +102,8 @@ def monitorUrls(stop_event): timeDiff = endTime - startTime print(timeDiff) + + logURLLog(socket.gethostname(), datetime.now(), baseUrl, timeDiff) else: urlFail = True @@ -110,8 +112,13 @@ def monitorUrls(stop_event): def logHostLog(hostname, logTime, cpu, memory): - manager = hostLogsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) - manager.insert_host_log(hostname, logTime, cpu, memory) + manager = logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) + manager.insertHostLog(hostname, socket.gethostbyname(socket.gethostname()), logTime, cpu, memory) + +def logURLLog(hostname, logTime, url, responseTime): + + manager = logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) + manager.insertURLLog(hostname, socket.gethostbyname(socket.gethostname()), logTime, url, responseTime) def main():