diff --git a/log.py b/log.py index 4778890..5dedf89 100644 --- a/log.py +++ b/log.py @@ -24,6 +24,8 @@ import pyodbc import time +import requests +from retrying import retry from odbcReferences import driver from config import logRetentionDays, maximumSQLAttempts, loggingMode from datetime import datetime, timedelta @@ -99,6 +101,155 @@ class logsManager: time.sleep(1) + @retry(stop_max_attempt_number=3, wait_fixed=1000) + def geoIPLookup(self, ip, token): + + global delayUntil + + try: + + if 'delayUntil' in locals() and datetime.now() < delayUntil: + + print("Rate limit exceeded. Please wait before trying again.") + return None + + url = f"https://ipinfo.io/{ip}?token={token}" + + response = requests.get(url) + response.raise_for_status() + data = response.json() + + if response.status_code == 429: + + print("Rate limit exceeded. Please wait before trying again.") + today = datetime.now() + + if today.month == 12: + + delayUntil = datetime(today.year + 1, 1, 1) + + else: + + delayUntil = datetime(today.year, today.month + 1, 1) + + return None + + geoinfo = { + "ip": data.get("ip"), + "hostname": data.get("hostname"), + "city": data.get("city"), + "region": data.get("region"), + "country": data.get("country"), + "loc": data.get("loc"), + "org": data.get("org"), + "postal": data.get("postal"), + "timezone": data.get("timezone"), + } + + + return geoinfo + + except Exception as e: + + print("An error occurred whilst doing a GeoIP lookup.", e) + raise + + def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token): + + currentAttempts = 1 + + self.insertHost(hostname, ipAddress) + self.deleteOldLogs("monutil_hostlogs", "logTime") + + while currentAttempts <= maximumSQLAttempts: + + try: + + conn = pyodbc.connect(self.conn_str) + cursor = conn.cursor() + + if int(live) == 0: + + if loggingMode == 'mssql' or loggingMode == 'mariadb': + cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail) + elif loggingMode == 'postgresql': + cursor.execute("UPDATE monutil_ipblock SET live = FALSE WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = TRUE", hostname, blockedIPAddress, jail) + else: + raise Exception("A serious error has occurred. Unrecognised DBMS.") + + autoIncrementID = 0 + + else: + + if loggingMode == 'mariadb': + + cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime) + cursor.execute("SELECT LAST_INSERT_ID()") + autoIncrementID = cursor.fetchone()[0] + + elif loggingMode == 'mssql': + + query = cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) OUTPUT INSERTED.logID VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime) + autoIncrementID = query.fetchone()[0] + + elif loggingMode == 'postgresql': + + cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING logID;", hostname, blockedIPAddress, jail, live, logTime) + autoIncrementID = cursor.fetchone()[0] + + else: + + raise Exception("A serious error has occurred. Unrecognised DBMS.") + + conn.commit() + conn.close() + + if autoIncrementID > 0: + + geoinfo = self.geoIPLookup(blockedIPAddress, token) + self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"], geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"]) + + + break + + except pyodbc.Error as ex: + + currentAttempts += 1 + print("Error inserting/updating data in monutil_ipblock:", ex) + + if not currentAttempts <= maximumSQLAttempts: + raise + + time.sleep(1) + + def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone): + + currentAttempts = 1 + + while currentAttempts <= maximumSQLAttempts: + + try: + conn = pyodbc.connect(self.conn_str) + cursor = conn.cursor() + + cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID, + hostname, city, region, country, loc, org, postal, timezone) + + conn.commit() + conn.close() + break + + except pyodbc.Error as ex: + + currentAttempts += 1 + print("Error inserting data in monutil_geoip:", ex) + + if not currentAttempts <= maximumSQLAttempts: + raise + + time.sleep(1) + + def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime): currentAttempts = 1 @@ -143,7 +294,7 @@ class logsManager: oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC" - elif loggingMode == 'mariadb': + elif loggingMode == 'mariadb' or loggingMode == 'postgresql': oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1" diff --git a/odbcReferences.py b/odbcReferences.py index 766d017..a581b6a 100644 --- a/odbcReferences.py +++ b/odbcReferences.py @@ -26,6 +26,7 @@ import config odbcMariaDB = '{MariaDB}' odbcMSSQL = '{ODBC Driver 17 for SQL Server}' +odbcPostgreSQL = '{PostgreSQL}' # DO NOT MODIFY ANYTHING BELOW THIS LINE @@ -38,3 +39,7 @@ if config.loggingMode == 'mariadb': elif config.loggingMode == 'mssql': driver = odbcMSSQL + +elif config.loggingMode == 'postgresql': + + driver = odbcPostgreSQL diff --git a/rabbitmq.py b/rabbitmq.py index 0b04047..fa5a5c8 100644 --- a/rabbitmq.py +++ b/rabbitmq.py @@ -75,19 +75,26 @@ class rabbitMQClient: result = body.split('|') if result[3] == 'cpumem': - + result[2] = datetime.datetime.fromisoformat(result[2]) manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager.insertHostLog(result[0], result[1], result[2], result[4], result[5]) ch.basic_ack(delivery_tag=method.delivery_tag) if result[3] == 'url': - + result[2] = datetime.datetime.fromisoformat(result[2]) manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager.insertURLLog(result[0], result[1], result[2], result[4], result[5]) ch.basic_ack(delivery_tag=method.delivery_tag) + if result[3] == 'ipBlock': + + result[2] = datetime.datetime.fromisoformat(result[2]) + manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) + manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken) + ch.basic_ack(delivery_tag=method.delivery_tag) + context = ssl.create_default_context( cafile=self.ca) context.verify_mode = ssl.CERT_REQUIRED diff --git a/reportIPBlock.py b/reportIPBlock.py new file mode 100644 index 0000000..2dafa8e --- /dev/null +++ b/reportIPBlock.py @@ -0,0 +1,116 @@ +def main(): + + import socket + import os + import config + import datetime + import signal + import threading + from log import logsManager + import sys + import select + + socketFile = '/etc/monutil/ip.socket' + + if len(sys.argv) == 4: + + hostname = socket.gethostname() + ipAddress = socket.gethostbyname(hostname) + blockedIPAddress = sys.argv[1] + jail = sys.argv[2] + live = int(sys.argv[3]) + logTime = datetime.datetime.now() + + clientSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + clientSocket.connect(socketFile) + data = f"{hostname}|{ipAddress}|{logTime}|ipBlock|{blockedIPAddress}|{jail}|{live}" + clientSocket.send(data.encode('utf-8')) + clientSocket.close() + sys.exit(0) + + else: + + if os.path.exists(socketFile): + os.remove(socketFile) + + stopEvent = threading.Event() + failureEvent = threading.Event() + + dataBuffer = [] + + def publishData(stopEvent): + + try: + + if config.loggingMode == 'rabbitmq': + + import rabbitmq + rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey) + + while not (stopEvent.is_set() and not failureEvent.is_set()): + + if dataBuffer: + + data = dataBuffer.pop(0) + + if config.loggingMode == 'rabbitmq': + + rabbitmq.publish(f"{data}") + + else: + + print("Not yet implemented") + + except Exception: + failureEvent.set() + + def cleanup(signum, frame): + + print("Signal received, shutting down...") + stopEvent.set() + + signal.signal(signal.SIGTERM, cleanup) + signal.signal(signal.SIGINT, cleanup) + + def server(stopEvent): + + try: + + serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + serverSocket.bind(socketFile) + serverSocket.listen(50) + + while not (stopEvent.is_set() and not failureEvent.is_set()): + + readable, _, _ = select.select([serverSocket], [], [], 1.0) + + if readable: + clientSocket, _ = serverSocket.accept() + data = clientSocket.recv(1024).decode('utf-8') + + if data: + + dataBuffer.append(data) + print(data) + + clientSocket.close() + + except Exception: + + failureEvent.set() + + publishThread = threading.Thread(target=publishData, args=(stopEvent,)) + serverThread = threading.Thread(target=server, args=(stopEvent,)) + + publishThread.start() + serverThread.start() + + publishThread.join() + serverThread.join() + + if failureEvent.is_set(): + print("One of the threads failed. Terminating") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/schema/mariadb.sql b/schema/mariadb.sql index 77ae561..e102b03 100644 --- a/schema/mariadb.sql +++ b/schema/mariadb.sql @@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts ( ); CREATE TABLE monutil_hostlogs ( - logID INT AUTO_INCREMENT PRIMARY KEY, + logID BIGINT AUTO_INCREMENT PRIMARY KEY, hostname VARCHAR(255) NOT NULL, logTime DATETIME NOT NULL, cpu DECIMAL(5, 2) NOT NULL, @@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs ( ); CREATE TABLE monutil_urlLogs ( - logID INT AUTO_INCREMENT PRIMARY KEY, + logID BIGINT AUTO_INCREMENT PRIMARY KEY, hostname VARCHAR(255) NOT NULL, url TEXT NOT NULL, logTime DATETIME NOT NULL, responseTime DECIMAL(5, 2) NOT NULL, FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) ); + +CREATE TABLE monutil_ipblock ( + logID BIGINT AUTO_INCREMENT PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + blockedIPAddress VARCHAR(45) NOT NULL, + jail VARCHAR(255), + live BOOLEAN NOT NULL, + logTime DATETIME NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_geoip ( + geoID BIGINT AUTO_INCREMENT PRIMARY KEY, + logID BIGINT NOT NULL, + hostname VARCHAR(255), + city VARCHAR(255), + region VARCHAR(255), + country VARCHAR(255), + loc VARCHAR(50), + org VARCHAR(255), + postal VARCHAR(20), + timezone VARCHAR(50), + FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) +); diff --git a/schema/mssql.sql b/schema/mssql.sql index 1ce2843..670ffb6 100644 --- a/schema/mssql.sql +++ b/schema/mssql.sql @@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts ( ); CREATE TABLE monutil_hostlogs ( - logID INT IDENTITY(1, 1) PRIMARY KEY, + logID BIGINT IDENTITY(1, 1) PRIMARY KEY, hostname NVARCHAR(255) NOT NULL, logTime DATETIME NOT NULL, cpu DECIMAL(5, 2) NOT NULL, @@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs ( ); CREATE TABLE monutil_urlLogs ( - logID INT IDENTITY(1, 1) PRIMARY KEY, + logID BIGINT IDENTITY(1, 1) PRIMARY KEY, hostname NVARCHAR(255) NOT NULL, url NVARCHAR(MAX) NOT NULL, logTime DATETIME NOT NULL, responseTime DECIMAL(5, 2) NOT NULL, FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) ); + +CREATE TABLE monutil_ipblock ( + logID BIGINT IDENTITY(1, 1) PRIMARY KEY, + hostname NVARCHAR(255) NOT NULL, + blockedIPAddress NVARCHAR(45) NOT NULL, + jail NVARCHAR(255), + live BIT NOT NULL, + logTime DATETIME NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_geoip ( + geoID BIGINT IDENTITY(1,1) PRIMARY KEY, + logID BIGINT NOT NULL, + hostname NVARCHAR(255), + city NVARCHAR(255), + region NVARCHAR(255), + country NVARCHAR(255), + loc NVARCHAR(50), + org NVARCHAR(255), + postal NVARCHAR(20), + timezone NVARCHAR(50), + FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) +); diff --git a/schema/postgresql.sql b/schema/postgresql.sql new file mode 100644 index 0000000..37655ee --- /dev/null +++ b/schema/postgresql.sql @@ -0,0 +1,46 @@ +CREATE TABLE monutil_hosts ( + hostname VARCHAR(255) PRIMARY KEY, + ipAddress VARCHAR(15) NOT NULL +); + +CREATE TABLE monutil_hostlogs ( + logID BIGSERIAL PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + logTime TIMESTAMP NOT NULL, + cpu DECIMAL(5, 2) NOT NULL, + memory DECIMAL(5, 2) NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_urlLogs ( + logID BIGSERIAL PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + url TEXT NOT NULL, + logTime TIMESTAMP NOT NULL, + responseTime DECIMAL(5, 2) NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_ipblock ( + logID BIGSERIAL PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + blockedIPAddress VARCHAR(45) NOT NULL, + jail VARCHAR(255), + live BOOLEAN NOT NULL, + logTime TIMESTAMP NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_geoip ( + geoID BIGSERIAL PRIMARY KEY, + logID BIGINT NOT NULL, + hostname VARCHAR(255), + city VARCHAR(255), + region VARCHAR(255), + country VARCHAR(255), + loc VARCHAR(50), + org VARCHAR(255), + postal VARCHAR(20), + timezone VARCHAR(50), + FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) +);