From 7f34688330a97cb2baf2b4dbd24adf3d752e4703 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Mon, 19 Aug 2024 16:25:22 +0100 Subject: [PATCH 01/12] Updated log class ready for IP block changes. Untested. --- log.py | 145 ++++++++++++++++++++++++++++++++++++++++++ schema/mariadb.sql | 28 +++++++- schema/mssql.sql | 28 +++++++- schema/postgresql.sql | 46 ++++++++++++++ 4 files changed, 243 insertions(+), 4 deletions(-) create mode 100644 schema/postgresql.sql diff --git a/log.py b/log.py index 4778890..5546511 100644 --- a/log.py +++ b/log.py @@ -24,6 +24,8 @@ import pyodbc import time +import requests +from retrying import retry from odbcReferences import driver from config import logRetentionDays, maximumSQLAttempts, loggingMode from datetime import datetime, timedelta @@ -99,6 +101,149 @@ class logsManager: time.sleep(1) + @retry(stop_max_attempt_number=3, wait_fixed=1000) + def geoIPLookup(self, ip, token): + + global delayUntil + + try: + + if 'delayUntil' in locals() and datetime.now() < delayUntil: + + print("Rate limit exceeded. Please wait before trying again.") + return None + + url = f"https://ipinfo.io/{ip}?token={token}" + + response = requests.get(url) + response.raise_for_status() + data = response.json() + + if response.status_code == 429: + + print("Rate limit exceeded. Please wait before trying again.") + today = datetime.now() + + if today.month == 12: + + delayUntil = datetime(today.year + 1, 1, 1) + + else: + + delayUntil = datetime(today.year, today.month + 1, 1) + + return None + + geoinfo = { + "ip": data.get("ip"), + "hostname": data.get("hostname"), + "city": data.get("city"), + "region": data.get("region"), + "country": data.get("country"), + "loc": data.get("loc"), + "org": data.get("org"), + "postal": data.get("postal"), + "timezone": data.get("timezone"), + } + + + return geoinfo + + except Exception as e: + + print("An error occurred whilst doing a GeoIP lookup.", e) + raise + + def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token): + + currentAttempts = 1 + + self.insertHost(hostname, ipAddress) + self.deleteOldLogs("monutil_hostlogs", "logTime") + + while currentAttempts <= maximumSQLAttempts: + + try: + + conn = pyodbc.connect(self.conn_str) + cursor = conn.cursor() + + if live == 0: + + cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail) + autoIncrementID = 0 + + else: + + if loggingMode == 'mariadb': + + cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime) + cursor.execute("SELECT LAST_INSERT_ID()") + + elif loggingMode == 'mssql': + + cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?); SELECT LAST_INSERT_ID();", hostname, blockedIPAddress, jail, live, logTime) + + elif loggingMode == 'postgresql': + + cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING id;", hostname, blockedIPAddress, jail, live, logTime) + + else: + + raise Exception("A serious error has occurred. Unrecognised DBMS.") + + + autoIncrementID = cursor.fetchone()[0] + + conn.commit() + conn.close() + + if autoIncrementID > 0: + + geoinfo = self.geoIPLookup(blockedIPAddress, token) + self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"], geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"]) + + + break + + except pyodbc.Error as ex: + + currentAttempts += 1 + print("Error inserting/updating data in monutil_ipblock:", ex) + + if not currentAttempts <= maximumSQLAttempts: + raise + + time.sleep(1) + + def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone): + + currentAttempts = 1 + + while currentAttempts <= maximumSQLAttempts: + + try: + conn = pyodbc.connect(self.conn_str) + cursor = conn.cursor() + + cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID, + hostname, city, region, country, loc, org, postal, timezone) + + conn.commit() + conn.close() + break + + except pyodbc.Error as ex: + + currentAttempts += 1 + print("Error inserting data in monutil_geoip:", ex) + + if not currentAttempts <= maximumSQLAttempts: + raise + + time.sleep(1) + + def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime): currentAttempts = 1 diff --git a/schema/mariadb.sql b/schema/mariadb.sql index 77ae561..25f006c 100644 --- a/schema/mariadb.sql +++ b/schema/mariadb.sql @@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts ( ); CREATE TABLE monutil_hostlogs ( - logID INT AUTO_INCREMENT PRIMARY KEY, + logID BIGINT AUTO_INCREMENT PRIMARY KEY, hostname VARCHAR(255) NOT NULL, logTime DATETIME NOT NULL, cpu DECIMAL(5, 2) NOT NULL, @@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs ( ); CREATE TABLE monutil_urlLogs ( - logID INT AUTO_INCREMENT PRIMARY KEY, + logID BIGINT AUTO_INCREMENT PRIMARY KEY, hostname VARCHAR(255) NOT NULL, url TEXT NOT NULL, logTime DATETIME NOT NULL, responseTime DECIMAL(5, 2) NOT NULL, FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) ); + +CREATE TABLE monutil_ipblock ( + logID BIGINT AUTO_INCREMENT PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + blockedIPAddress VARCHAR(45) NOT NULL, + jail VARCHAR(255), + live BOOLEAN NOT NULL, + logTime DATETIME NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_geoip ( + geoID BIGINT AUTO_INCREMENT PRIMARY KEY, + logID BIGINT NOT NULL, + hostname VARCHAR(255) NOT NULL, + city VARCHAR(255), + region VARCHAR(255), + country VARCHAR(255), + loc VARCHAR(50), + org VARCHAR(255), + postal VARCHAR(20), + timezone VARCHAR(50), + FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) +); diff --git a/schema/mssql.sql b/schema/mssql.sql index 1ce2843..b49676c 100644 --- a/schema/mssql.sql +++ b/schema/mssql.sql @@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts ( ); CREATE TABLE monutil_hostlogs ( - logID INT IDENTITY(1, 1) PRIMARY KEY, + logID BIGINT IDENTITY(1, 1) PRIMARY KEY, hostname NVARCHAR(255) NOT NULL, logTime DATETIME NOT NULL, cpu DECIMAL(5, 2) NOT NULL, @@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs ( ); CREATE TABLE monutil_urlLogs ( - logID INT IDENTITY(1, 1) PRIMARY KEY, + logID BIGINT IDENTITY(1, 1) PRIMARY KEY, hostname NVARCHAR(255) NOT NULL, url NVARCHAR(MAX) NOT NULL, logTime DATETIME NOT NULL, responseTime DECIMAL(5, 2) NOT NULL, FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) ); + +CREATE TABLE monutil_ipblock ( + logID BIGINT IDENTITY(1, 1) PRIMARY KEY, + hostname NVARCHAR(255) NOT NULL, + blockedIPAddress NVARCHAR(45) NOT NULL, + jail NVARCHAR(255), + live BIT NOT NULL, + logTime DATETIME NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_geoip ( + geoID BIGINT IDENTITY(1,1) PRIMARY KEY, + logID BIGINT NOT NULL, + hostname NVARCHAR(255) NOT NULL, + city NVARCHAR(255), + region NVARCHAR(255), + country NVARCHAR(255), + loc NVARCHAR(50), + org NVARCHAR(255), + postal NVARCHAR(20), + timezone NVARCHAR(50), + FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) +); diff --git a/schema/postgresql.sql b/schema/postgresql.sql new file mode 100644 index 0000000..581aa51 --- /dev/null +++ b/schema/postgresql.sql @@ -0,0 +1,46 @@ +CREATE TABLE monutil_hosts ( + hostname VARCHAR(255) PRIMARY KEY, + ipAddress VARCHAR(15) NOT NULL +); + +CREATE TABLE monutil_hostlogs ( + logID BIGSERIAL PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + logTime TIMESTAMP NOT NULL, + cpu DECIMAL(5, 2) NOT NULL, + memory DECIMAL(5, 2) NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_urlLogs ( + logID BIGSERIAL PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + url TEXT NOT NULL, + logTime TIMESTAMP NOT NULL, + responseTime DECIMAL(5, 2) NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_ipblock ( + logID BIGSERIAL PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + blockedIPAddress VARCHAR(45) NOT NULL, + jail VARCHAR(255), + live BOOLEAN NOT NULL, + logTime TIMESTAMP NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_geoip ( + geoID BIGSERIAL PRIMARY KEY, + logID BIGINT NOT NULL, + hostname VARCHAR(255) NOT NULL, + city VARCHAR(255), + region VARCHAR(255), + country VARCHAR(255), + loc VARCHAR(50), + org VARCHAR(255), + postal VARCHAR(20), + timezone VARCHAR(50), + FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) +); From 20a2a2be265ab53e0b10ff5f7deb6257040d75e8 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Mon, 19 Aug 2024 17:36:43 +0100 Subject: [PATCH 02/12] Some further changes following testing across all three DBMSs --- log.py | 28 +++++++++++++++++----------- odbcReferences.py | 5 +++++ reportIPBlock.py | 27 +++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 reportIPBlock.py diff --git a/log.py b/log.py index 5546511..db723fc 100644 --- a/log.py +++ b/log.py @@ -155,7 +155,7 @@ class logsManager: raise def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token): - + currentAttempts = 1 self.insertHost(hostname, ipAddress) @@ -167,39 +167,45 @@ class logsManager: conn = pyodbc.connect(self.conn_str) cursor = conn.cursor() - + if live == 0: - cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail) + if loggingMode == 'mssql' or loggingMode == 'mariadb': + cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail) + elif loggingMode == 'postgresql': + cursor.execute("UPDATE monutil_ipblock SET live = FALSE WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = TRUE", hostname, blockedIPAddress, jail) + else: + raise Exception("A serious error has occurred. Unrecognised DBMS.") + autoIncrementID = 0 else: - + if loggingMode == 'mariadb': cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime) cursor.execute("SELECT LAST_INSERT_ID()") + autoIncrementID = cursor.fetchone()[0] elif loggingMode == 'mssql': - cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?); SELECT LAST_INSERT_ID();", hostname, blockedIPAddress, jail, live, logTime) + query = cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) OUTPUT INSERTED.logID VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime) + autoIncrementID = query.fetchone()[0] elif loggingMode == 'postgresql': - cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING id;", hostname, blockedIPAddress, jail, live, logTime) + cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING logID;", hostname, blockedIPAddress, jail, live, logTime) + autoIncrementID = cursor.fetchone()[0] else: raise Exception("A serious error has occurred. Unrecognised DBMS.") - - autoIncrementID = cursor.fetchone()[0] - conn.commit() conn.close() if autoIncrementID > 0: - + geoinfo = self.geoIPLookup(blockedIPAddress, token) self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"], geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"]) @@ -288,7 +294,7 @@ class logsManager: oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC" - elif loggingMode == 'mariadb': + elif loggingMode == 'mariadb' or loggingMode == 'postgresql': oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1" diff --git a/odbcReferences.py b/odbcReferences.py index 766d017..a581b6a 100644 --- a/odbcReferences.py +++ b/odbcReferences.py @@ -26,6 +26,7 @@ import config odbcMariaDB = '{MariaDB}' odbcMSSQL = '{ODBC Driver 17 for SQL Server}' +odbcPostgreSQL = '{PostgreSQL}' # DO NOT MODIFY ANYTHING BELOW THIS LINE @@ -38,3 +39,7 @@ if config.loggingMode == 'mariadb': elif config.loggingMode == 'mssql': driver = odbcMSSQL + +elif config.loggingMode == 'postgresql': + + driver = odbcPostgreSQL diff --git a/reportIPBlock.py b/reportIPBlock.py new file mode 100644 index 0000000..f6a01c4 --- /dev/null +++ b/reportIPBlock.py @@ -0,0 +1,27 @@ +def main(): + + import socket + import config + import datetime + from log import logsManager + import sys + + if len(sys.argv) != 4: + print("Usage: python script.py ") + sys.exit(1) + + manager = logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) + hostname = socket.gethostname() + ipAddress = socket.gethostbyname(hostname) + blockedIPAddress = sys.argv[1] + jail = sys.argv[2] + live = int(sys.argv[3]) + logTime = datetime.datetime.now() + token = config.ipinfoAPIToken + + print(hostname, ipAddress, blockedIPAddress, jail, live, logTime, token) + + manager.insertIPBlock(hostname, ipAddress, blockedIPAddress, jail, live, logTime, token) + +if __name__ == "__main__": + main() From 46d895d9004e97fa2e256126b9ac59dcc1fb9e8d Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Mon, 19 Aug 2024 18:08:36 +0100 Subject: [PATCH 03/12] Added some RabbitMQ amendments. --- rabbitmq.py | 6 ++++++ reportIPBlock.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/rabbitmq.py b/rabbitmq.py index 0b04047..5cade99 100644 --- a/rabbitmq.py +++ b/rabbitmq.py @@ -88,6 +88,12 @@ class rabbitMQClient: manager.insertURLLog(result[0], result[1], result[2], result[4], result[5]) ch.basic_ack(delivery_tag=method.delivery_tag) + if result[3] == 'ipBlock': + + result[2] = datetime.datetime.fromisoformat(result[2]) + manager = logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) + manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken) + context = ssl.create_default_context( cafile=self.ca) context.verify_mode = ssl.CERT_REQUIRED diff --git a/reportIPBlock.py b/reportIPBlock.py index f6a01c4..cfcf352 100644 --- a/reportIPBlock.py +++ b/reportIPBlock.py @@ -10,7 +10,6 @@ def main(): print("Usage: python script.py ") sys.exit(1) - manager = logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) hostname = socket.gethostname() ipAddress = socket.gethostbyname(hostname) blockedIPAddress = sys.argv[1] @@ -20,8 +19,15 @@ def main(): token = config.ipinfoAPIToken print(hostname, ipAddress, blockedIPAddress, jail, live, logTime, token) + + if not config.loggingMode == 'none' and not config.loggingMode == 'rabbitmq': - manager.insertIPBlock(hostname, ipAddress, blockedIPAddress, jail, live, logTime, token) + manager = logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) + manager.insertIPBlock(hostname, ipAddress, blockedIPAddress, jail, live, logTime, token) + + if config.loggingMode == 'rabbitmq': + + rabbitmq.publish(hostname + '|' + socket.gethostbyname(socket.gethostname()) + '|' + str(logTime) + '|' + 'ipBlock' + '|' + str(blockedIPAddress) + '|' + str(jail) + '|' str(live) if __name__ == "__main__": main() From 2e04b5eeb8d027976f2458d2a1d76a6d8cc9a6fe Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Mon, 19 Aug 2024 18:56:56 +0100 Subject: [PATCH 04/12] Defect fixes following testing. --- rabbitmq.py | 8 ++++---- reportIPBlock.py | 10 +++++----- schema/mariadb.sql | 2 +- schema/mssql.sql | 2 +- schema/postgresql.sql | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/rabbitmq.py b/rabbitmq.py index 5cade99..2d78621 100644 --- a/rabbitmq.py +++ b/rabbitmq.py @@ -75,23 +75,23 @@ class rabbitMQClient: result = body.split('|') if result[3] == 'cpumem': - + result[2] = datetime.datetime.fromisoformat(result[2]) manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager.insertHostLog(result[0], result[1], result[2], result[4], result[5]) ch.basic_ack(delivery_tag=method.delivery_tag) if result[3] == 'url': - + result[2] = datetime.datetime.fromisoformat(result[2]) manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager.insertURLLog(result[0], result[1], result[2], result[4], result[5]) ch.basic_ack(delivery_tag=method.delivery_tag) if result[3] == 'ipBlock': - + result[2] = datetime.datetime.fromisoformat(result[2]) - manager = logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) + manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken) context = ssl.create_default_context( diff --git a/reportIPBlock.py b/reportIPBlock.py index cfcf352..37b46eb 100644 --- a/reportIPBlock.py +++ b/reportIPBlock.py @@ -16,9 +16,6 @@ def main(): jail = sys.argv[2] live = int(sys.argv[3]) logTime = datetime.datetime.now() - token = config.ipinfoAPIToken - - print(hostname, ipAddress, blockedIPAddress, jail, live, logTime, token) if not config.loggingMode == 'none' and not config.loggingMode == 'rabbitmq': @@ -26,8 +23,11 @@ def main(): manager.insertIPBlock(hostname, ipAddress, blockedIPAddress, jail, live, logTime, token) if config.loggingMode == 'rabbitmq': - - rabbitmq.publish(hostname + '|' + socket.gethostbyname(socket.gethostname()) + '|' + str(logTime) + '|' + 'ipBlock' + '|' + str(blockedIPAddress) + '|' + str(jail) + '|' str(live) + + import rabbitmq + + rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey) + rabbitmq.publish(hostname + '|' + socket.gethostbyname(socket.gethostname()) + '|' + str(logTime) + '|' + 'ipBlock' + '|' + str(blockedIPAddress) + '|' + str(jail) + '|' + str(live)) if __name__ == "__main__": main() diff --git a/schema/mariadb.sql b/schema/mariadb.sql index 25f006c..e102b03 100644 --- a/schema/mariadb.sql +++ b/schema/mariadb.sql @@ -34,7 +34,7 @@ CREATE TABLE monutil_ipblock ( CREATE TABLE monutil_geoip ( geoID BIGINT AUTO_INCREMENT PRIMARY KEY, logID BIGINT NOT NULL, - hostname VARCHAR(255) NOT NULL, + hostname VARCHAR(255), city VARCHAR(255), region VARCHAR(255), country VARCHAR(255), diff --git a/schema/mssql.sql b/schema/mssql.sql index b49676c..670ffb6 100644 --- a/schema/mssql.sql +++ b/schema/mssql.sql @@ -34,7 +34,7 @@ CREATE TABLE monutil_ipblock ( CREATE TABLE monutil_geoip ( geoID BIGINT IDENTITY(1,1) PRIMARY KEY, logID BIGINT NOT NULL, - hostname NVARCHAR(255) NOT NULL, + hostname NVARCHAR(255), city NVARCHAR(255), region NVARCHAR(255), country NVARCHAR(255), diff --git a/schema/postgresql.sql b/schema/postgresql.sql index 581aa51..37655ee 100644 --- a/schema/postgresql.sql +++ b/schema/postgresql.sql @@ -34,7 +34,7 @@ CREATE TABLE monutil_ipblock ( CREATE TABLE monutil_geoip ( geoID BIGSERIAL PRIMARY KEY, logID BIGINT NOT NULL, - hostname VARCHAR(255) NOT NULL, + hostname VARCHAR(255), city VARCHAR(255), region VARCHAR(255), country VARCHAR(255), From 8ce56e19bedf1f781f69d81b5cd89843aff519b7 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Mon, 19 Aug 2024 22:44:26 +0100 Subject: [PATCH 05/12] Some further changes to assist with bottleknecking causing upstream issues to UFW --- reportIPBlock.py | 97 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 18 deletions(-) diff --git a/reportIPBlock.py b/reportIPBlock.py index 37b46eb..0f2fe35 100644 --- a/reportIPBlock.py +++ b/reportIPBlock.py @@ -1,33 +1,94 @@ def main(): import socket + import os import config import datetime + import signal + import threading from log import logsManager import sys - - if len(sys.argv) != 4: - print("Usage: python script.py ") - sys.exit(1) + import select - hostname = socket.gethostname() - ipAddress = socket.gethostbyname(hostname) - blockedIPAddress = sys.argv[1] - jail = sys.argv[2] - live = int(sys.argv[3]) - logTime = datetime.datetime.now() - - if not config.loggingMode == 'none' and not config.loggingMode == 'rabbitmq': + socketFile = '/etc/monutil/ip.socket' - manager = logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) - manager.insertIPBlock(hostname, ipAddress, blockedIPAddress, jail, live, logTime, token) + if len(sys.argv) == 4: - if config.loggingMode == 'rabbitmq': + hostname = socket.gethostname() + ipAddress = socket.gethostbyname(hostname) + blockedIPAddress = sys.argv[1] + jail = sys.argv[2] + live = int(sys.argv[3]) + logTime = datetime.datetime.now() + + clientSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + clientSocket.connect(socketFile) + data = f"{hostname}|{ipAddress}|{logTime}|ipBlock|{blockedIPAddress}|{jail}|{live}" + clientSocket.send(data.encode('utf-8')) + clientSocket.close() + sys.exit(0) + + else: - import rabbitmq + if os.path.exists(socketFile): + os.remove(socketFile) + + stop_event = threading.Event() + dataBuffer = [] + + def publishData(stop_event): + + if config.loggingMode = 'rabbitmq': + + import rabbitmq + rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey) + + while not (stop_event.is_set()): + if dataBuffer: + data = dataBuffer.pop(0) + + if config.loggingMode = 'rabbitmq': + rabbitmq.publish(f"{data}") + else: + print("Not yet implemented") + + def cleanup(signum, frame): + + print("Signal received, shutting down...") + stop_event.set() + + signal.signal(signal.SIGTERM, cleanup) + signal.signal(signal.SIGINT, cleanup) + + def server(stop_event): + + serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + serverSocket.bind(socketFile) + serverSocket.listen(50) + + while not (stop_event.is_set()): + + readable, _, _ = select.select([serverSocket], [], [], 1.0) + + if readable: + clientSocket, _ = serverSocket.accept() + data = clientSocket.recv(1024).decode('utf-8') + + if data: + + dataBuffer.append(data) + print(data) + + clientSocket.close() + + publishThread = threading.Thread(target=publishData, args=(stop_event,)) + serverThread = threading.Thread(target=server, args=(stop_event,)) - rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey) - rabbitmq.publish(hostname + '|' + socket.gethostbyname(socket.gethostname()) + '|' + str(logTime) + '|' + 'ipBlock' + '|' + str(blockedIPAddress) + '|' + str(jail) + '|' + str(live)) + publishThread.start() + serverThread.start() + + publishThread.join() + serverThread.join() if __name__ == "__main__": main() From 387c0daab003fcba0414b3235f17dacc71d31d69 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Mon, 19 Aug 2024 22:49:49 +0100 Subject: [PATCH 06/12] Fixed silly mistake --- reportIPBlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportIPBlock.py b/reportIPBlock.py index 0f2fe35..01324bc 100644 --- a/reportIPBlock.py +++ b/reportIPBlock.py @@ -47,7 +47,7 @@ def main(): if dataBuffer: data = dataBuffer.pop(0) - if config.loggingMode = 'rabbitmq': + if config.loggingMode == 'rabbitmq': rabbitmq.publish(f"{data}") else: print("Not yet implemented") From daa88b46a91ce346bdb5f668eecee5e2b4149c1b Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Mon, 19 Aug 2024 22:51:53 +0100 Subject: [PATCH 07/12] Fixed silly mistake --- reportIPBlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportIPBlock.py b/reportIPBlock.py index 01324bc..d80a036 100644 --- a/reportIPBlock.py +++ b/reportIPBlock.py @@ -38,7 +38,7 @@ def main(): def publishData(stop_event): - if config.loggingMode = 'rabbitmq': + if config.loggingMode == 'rabbitmq': import rabbitmq rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey) From c11943cc67b9ec31c51cafaa066c3f3cedc62ef9 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Mon, 19 Aug 2024 23:41:41 +0100 Subject: [PATCH 08/12] Defect fix --- rabbitmq.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rabbitmq.py b/rabbitmq.py index 2d78621..fa5a5c8 100644 --- a/rabbitmq.py +++ b/rabbitmq.py @@ -93,6 +93,7 @@ class rabbitMQClient: result[2] = datetime.datetime.fromisoformat(result[2]) manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken) + ch.basic_ack(delivery_tag=method.delivery_tag) context = ssl.create_default_context( cafile=self.ca) From b599a4a483efc4470c81480939873207e12ef154 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Tue, 20 Aug 2024 11:04:23 +0100 Subject: [PATCH 09/12] Added exceptioning handling --- reportIPBlock.py | 82 ++++++++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/reportIPBlock.py b/reportIPBlock.py index d80a036..090b8f3 100644 --- a/reportIPBlock.py +++ b/reportIPBlock.py @@ -33,56 +33,74 @@ def main(): if os.path.exists(socketFile): os.remove(socketFile) - stop_event = threading.Event() + stopEvent = threading.Event() + failureEvent = threading.Event() + dataBuffer = [] - def publishData(stop_event): + def publishData(stopEvent): - if config.loggingMode == 'rabbitmq': + try: - import rabbitmq - rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey) + if config.loggingMode == 'rabbitmq': + + import rabbitmq + rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey) - while not (stop_event.is_set()): - if dataBuffer: - data = dataBuffer.pop(0) + while not (stopEvent.is_set() and not failureEvent.is_set()): + + if dataBuffer: + + data = dataBuffer.pop(0) - if config.loggingMode == 'rabbitmq': - rabbitmq.publish(f"{data}") - else: - print("Not yet implemented") + if config.loggingMode == 'rabbitmq': + + rabbitmq.publish(f"{data}") + + else: + + print("Not yet implemented") + + except Exception: + failureEvent.set() def cleanup(signum, frame): print("Signal received, shutting down...") - stop_event.set() + stopEvent.set() signal.signal(signal.SIGTERM, cleanup) signal.signal(signal.SIGINT, cleanup) - def server(stop_event): + def server(stopEvent): - serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - serverSocket.bind(socketFile) - serverSocket.listen(50) - - while not (stop_event.is_set()): - - readable, _, _ = select.select([serverSocket], [], [], 1.0) + try: - if readable: - clientSocket, _ = serverSocket.accept() - data = clientSocket.recv(1024).decode('utf-8') + serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + serverSocket.bind(socketFile) + serverSocket.listen(50) + + while not (stopEvent.is_set() and not failureEvent.is_set()): + + readable, _, _ = select.select([serverSocket], [], [], 1.0) + + if readable: + clientSocket, _ = serverSocket.accept() + data = clientSocket.recv(1024).decode('utf-8') - if data: + if data: - dataBuffer.append(data) - print(data) + dataBuffer.append(data) + print(data) - clientSocket.close() + clientSocket.close() - publishThread = threading.Thread(target=publishData, args=(stop_event,)) - serverThread = threading.Thread(target=server, args=(stop_event,)) + except Exception: + + failureEvent.set() + + publishThread = threading.Thread(target=publishData, args=(stopEvent,)) + serverThread = threading.Thread(target=server, args=(stopEvent,)) publishThread.start() serverThread.start() @@ -90,5 +108,9 @@ def main(): publishThread.join() serverThread.join() + if failure_event.is_set(): + print("One of the threads failed. Terminating") + sys.exit(1) + if __name__ == "__main__": main() From bf82596b00d01b9c11450aa90e5fa30b565f7019 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Tue, 20 Aug 2024 11:35:31 +0100 Subject: [PATCH 10/12] Fix --- reportIPBlock.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/reportIPBlock.py b/reportIPBlock.py index 090b8f3..774cac3 100644 --- a/reportIPBlock.py +++ b/reportIPBlock.py @@ -53,13 +53,13 @@ def main(): data = dataBuffer.pop(0) - if config.loggingMode == 'rabbitmq': + if config.loggingMode == 'rabbitmq': - rabbitmq.publish(f"{data}") + rabbitmq.publish(f"{data}") - else: + else: - print("Not yet implemented") + print("Not yet implemented") except Exception: failureEvent.set() From e92d76cafdf0ccb4fc23a2e0e780a4722d1e3190 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Tue, 20 Aug 2024 11:36:07 +0100 Subject: [PATCH 11/12] Fix --- reportIPBlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportIPBlock.py b/reportIPBlock.py index 774cac3..2dafa8e 100644 --- a/reportIPBlock.py +++ b/reportIPBlock.py @@ -108,7 +108,7 @@ def main(): publishThread.join() serverThread.join() - if failure_event.is_set(): + if failureEvent.is_set(): print("One of the threads failed. Terminating") sys.exit(1) From ef294aa1c5d41f900790ba807cfcf545061d2d4c Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Tue, 20 Aug 2024 11:42:15 +0100 Subject: [PATCH 12/12] Defect Fix --- log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log.py b/log.py index db723fc..5dedf89 100644 --- a/log.py +++ b/log.py @@ -168,7 +168,7 @@ class logsManager: conn = pyodbc.connect(self.conn_str) cursor = conn.cursor() - if live == 0: + if int(live) == 0: if loggingMode == 'mssql' or loggingMode == 'mariadb': cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail)