diff --git a/log.py b/log.py index 5dedf89..4778890 100644 --- a/log.py +++ b/log.py @@ -24,8 +24,6 @@ import pyodbc import time -import requests -from retrying import retry from odbcReferences import driver from config import logRetentionDays, maximumSQLAttempts, loggingMode from datetime import datetime, timedelta @@ -101,155 +99,6 @@ class logsManager: time.sleep(1) - @retry(stop_max_attempt_number=3, wait_fixed=1000) - def geoIPLookup(self, ip, token): - - global delayUntil - - try: - - if 'delayUntil' in locals() and datetime.now() < delayUntil: - - print("Rate limit exceeded. Please wait before trying again.") - return None - - url = f"https://ipinfo.io/{ip}?token={token}" - - response = requests.get(url) - response.raise_for_status() - data = response.json() - - if response.status_code == 429: - - print("Rate limit exceeded. Please wait before trying again.") - today = datetime.now() - - if today.month == 12: - - delayUntil = datetime(today.year + 1, 1, 1) - - else: - - delayUntil = datetime(today.year, today.month + 1, 1) - - return None - - geoinfo = { - "ip": data.get("ip"), - "hostname": data.get("hostname"), - "city": data.get("city"), - "region": data.get("region"), - "country": data.get("country"), - "loc": data.get("loc"), - "org": data.get("org"), - "postal": data.get("postal"), - "timezone": data.get("timezone"), - } - - - return geoinfo - - except Exception as e: - - print("An error occurred whilst doing a GeoIP lookup.", e) - raise - - def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token): - - currentAttempts = 1 - - self.insertHost(hostname, ipAddress) - self.deleteOldLogs("monutil_hostlogs", "logTime") - - while currentAttempts <= maximumSQLAttempts: - - try: - - conn = pyodbc.connect(self.conn_str) - cursor = conn.cursor() - - if int(live) == 0: - - if loggingMode == 'mssql' or loggingMode == 'mariadb': - cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail) - elif loggingMode == 'postgresql': - cursor.execute("UPDATE monutil_ipblock SET live = FALSE WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = TRUE", hostname, blockedIPAddress, jail) - else: - raise Exception("A serious error has occurred. Unrecognised DBMS.") - - autoIncrementID = 0 - - else: - - if loggingMode == 'mariadb': - - cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime) - cursor.execute("SELECT LAST_INSERT_ID()") - autoIncrementID = cursor.fetchone()[0] - - elif loggingMode == 'mssql': - - query = cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) OUTPUT INSERTED.logID VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime) - autoIncrementID = query.fetchone()[0] - - elif loggingMode == 'postgresql': - - cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING logID;", hostname, blockedIPAddress, jail, live, logTime) - autoIncrementID = cursor.fetchone()[0] - - else: - - raise Exception("A serious error has occurred. Unrecognised DBMS.") - - conn.commit() - conn.close() - - if autoIncrementID > 0: - - geoinfo = self.geoIPLookup(blockedIPAddress, token) - self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"], geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"]) - - - break - - except pyodbc.Error as ex: - - currentAttempts += 1 - print("Error inserting/updating data in monutil_ipblock:", ex) - - if not currentAttempts <= maximumSQLAttempts: - raise - - time.sleep(1) - - def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone): - - currentAttempts = 1 - - while currentAttempts <= maximumSQLAttempts: - - try: - conn = pyodbc.connect(self.conn_str) - cursor = conn.cursor() - - cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID, - hostname, city, region, country, loc, org, postal, timezone) - - conn.commit() - conn.close() - break - - except pyodbc.Error as ex: - - currentAttempts += 1 - print("Error inserting data in monutil_geoip:", ex) - - if not currentAttempts <= maximumSQLAttempts: - raise - - time.sleep(1) - - def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime): currentAttempts = 1 @@ -294,7 +143,7 @@ class logsManager: oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC" - elif loggingMode == 'mariadb' or loggingMode == 'postgresql': + elif loggingMode == 'mariadb': oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1" diff --git a/odbcReferences.py b/odbcReferences.py index a581b6a..766d017 100644 --- a/odbcReferences.py +++ b/odbcReferences.py @@ -26,7 +26,6 @@ import config odbcMariaDB = '{MariaDB}' odbcMSSQL = '{ODBC Driver 17 for SQL Server}' -odbcPostgreSQL = '{PostgreSQL}' # DO NOT MODIFY ANYTHING BELOW THIS LINE @@ -39,7 +38,3 @@ if config.loggingMode == 'mariadb': elif config.loggingMode == 'mssql': driver = odbcMSSQL - -elif config.loggingMode == 'postgresql': - - driver = odbcPostgreSQL diff --git a/rabbitmq.py b/rabbitmq.py index fa5a5c8..0b04047 100644 --- a/rabbitmq.py +++ b/rabbitmq.py @@ -75,26 +75,19 @@ class rabbitMQClient: result = body.split('|') if result[3] == 'cpumem': - + result[2] = datetime.datetime.fromisoformat(result[2]) manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager.insertHostLog(result[0], result[1], result[2], result[4], result[5]) ch.basic_ack(delivery_tag=method.delivery_tag) if result[3] == 'url': - + result[2] = datetime.datetime.fromisoformat(result[2]) manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager.insertURLLog(result[0], result[1], result[2], result[4], result[5]) ch.basic_ack(delivery_tag=method.delivery_tag) - if result[3] == 'ipBlock': - - result[2] = datetime.datetime.fromisoformat(result[2]) - manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) - manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken) - ch.basic_ack(delivery_tag=method.delivery_tag) - context = ssl.create_default_context( cafile=self.ca) context.verify_mode = ssl.CERT_REQUIRED diff --git a/reportIPBlock.py b/reportIPBlock.py deleted file mode 100644 index 2dafa8e..0000000 --- a/reportIPBlock.py +++ /dev/null @@ -1,116 +0,0 @@ -def main(): - - import socket - import os - import config - import datetime - import signal - import threading - from log import logsManager - import sys - import select - - socketFile = '/etc/monutil/ip.socket' - - if len(sys.argv) == 4: - - hostname = socket.gethostname() - ipAddress = socket.gethostbyname(hostname) - blockedIPAddress = sys.argv[1] - jail = sys.argv[2] - live = int(sys.argv[3]) - logTime = datetime.datetime.now() - - clientSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - clientSocket.connect(socketFile) - data = f"{hostname}|{ipAddress}|{logTime}|ipBlock|{blockedIPAddress}|{jail}|{live}" - clientSocket.send(data.encode('utf-8')) - clientSocket.close() - sys.exit(0) - - else: - - if os.path.exists(socketFile): - os.remove(socketFile) - - stopEvent = threading.Event() - failureEvent = threading.Event() - - dataBuffer = [] - - def publishData(stopEvent): - - try: - - if config.loggingMode == 'rabbitmq': - - import rabbitmq - rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey) - - while not (stopEvent.is_set() and not failureEvent.is_set()): - - if dataBuffer: - - data = dataBuffer.pop(0) - - if config.loggingMode == 'rabbitmq': - - rabbitmq.publish(f"{data}") - - else: - - print("Not yet implemented") - - except Exception: - failureEvent.set() - - def cleanup(signum, frame): - - print("Signal received, shutting down...") - stopEvent.set() - - signal.signal(signal.SIGTERM, cleanup) - signal.signal(signal.SIGINT, cleanup) - - def server(stopEvent): - - try: - - serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - serverSocket.bind(socketFile) - serverSocket.listen(50) - - while not (stopEvent.is_set() and not failureEvent.is_set()): - - readable, _, _ = select.select([serverSocket], [], [], 1.0) - - if readable: - clientSocket, _ = serverSocket.accept() - data = clientSocket.recv(1024).decode('utf-8') - - if data: - - dataBuffer.append(data) - print(data) - - clientSocket.close() - - except Exception: - - failureEvent.set() - - publishThread = threading.Thread(target=publishData, args=(stopEvent,)) - serverThread = threading.Thread(target=server, args=(stopEvent,)) - - publishThread.start() - serverThread.start() - - publishThread.join() - serverThread.join() - - if failureEvent.is_set(): - print("One of the threads failed. Terminating") - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/schema/mariadb.sql b/schema/mariadb.sql index e102b03..77ae561 100644 --- a/schema/mariadb.sql +++ b/schema/mariadb.sql @@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts ( ); CREATE TABLE monutil_hostlogs ( - logID BIGINT AUTO_INCREMENT PRIMARY KEY, + logID INT AUTO_INCREMENT PRIMARY KEY, hostname VARCHAR(255) NOT NULL, logTime DATETIME NOT NULL, cpu DECIMAL(5, 2) NOT NULL, @@ -13,34 +13,10 @@ CREATE TABLE monutil_hostlogs ( ); CREATE TABLE monutil_urlLogs ( - logID BIGINT AUTO_INCREMENT PRIMARY KEY, + logID INT AUTO_INCREMENT PRIMARY KEY, hostname VARCHAR(255) NOT NULL, url TEXT NOT NULL, logTime DATETIME NOT NULL, responseTime DECIMAL(5, 2) NOT NULL, FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) ); - -CREATE TABLE monutil_ipblock ( - logID BIGINT AUTO_INCREMENT PRIMARY KEY, - hostname VARCHAR(255) NOT NULL, - blockedIPAddress VARCHAR(45) NOT NULL, - jail VARCHAR(255), - live BOOLEAN NOT NULL, - logTime DATETIME NOT NULL, - FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) -); - -CREATE TABLE monutil_geoip ( - geoID BIGINT AUTO_INCREMENT PRIMARY KEY, - logID BIGINT NOT NULL, - hostname VARCHAR(255), - city VARCHAR(255), - region VARCHAR(255), - country VARCHAR(255), - loc VARCHAR(50), - org VARCHAR(255), - postal VARCHAR(20), - timezone VARCHAR(50), - FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) -); diff --git a/schema/mssql.sql b/schema/mssql.sql index 670ffb6..1ce2843 100644 --- a/schema/mssql.sql +++ b/schema/mssql.sql @@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts ( ); CREATE TABLE monutil_hostlogs ( - logID BIGINT IDENTITY(1, 1) PRIMARY KEY, + logID INT IDENTITY(1, 1) PRIMARY KEY, hostname NVARCHAR(255) NOT NULL, logTime DATETIME NOT NULL, cpu DECIMAL(5, 2) NOT NULL, @@ -13,34 +13,10 @@ CREATE TABLE monutil_hostlogs ( ); CREATE TABLE monutil_urlLogs ( - logID BIGINT IDENTITY(1, 1) PRIMARY KEY, + logID INT IDENTITY(1, 1) PRIMARY KEY, hostname NVARCHAR(255) NOT NULL, url NVARCHAR(MAX) NOT NULL, logTime DATETIME NOT NULL, responseTime DECIMAL(5, 2) NOT NULL, FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) ); - -CREATE TABLE monutil_ipblock ( - logID BIGINT IDENTITY(1, 1) PRIMARY KEY, - hostname NVARCHAR(255) NOT NULL, - blockedIPAddress NVARCHAR(45) NOT NULL, - jail NVARCHAR(255), - live BIT NOT NULL, - logTime DATETIME NOT NULL, - FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) -); - -CREATE TABLE monutil_geoip ( - geoID BIGINT IDENTITY(1,1) PRIMARY KEY, - logID BIGINT NOT NULL, - hostname NVARCHAR(255), - city NVARCHAR(255), - region NVARCHAR(255), - country NVARCHAR(255), - loc NVARCHAR(50), - org NVARCHAR(255), - postal NVARCHAR(20), - timezone NVARCHAR(50), - FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) -); diff --git a/schema/postgresql.sql b/schema/postgresql.sql deleted file mode 100644 index 37655ee..0000000 --- a/schema/postgresql.sql +++ /dev/null @@ -1,46 +0,0 @@ -CREATE TABLE monutil_hosts ( - hostname VARCHAR(255) PRIMARY KEY, - ipAddress VARCHAR(15) NOT NULL -); - -CREATE TABLE monutil_hostlogs ( - logID BIGSERIAL PRIMARY KEY, - hostname VARCHAR(255) NOT NULL, - logTime TIMESTAMP NOT NULL, - cpu DECIMAL(5, 2) NOT NULL, - memory DECIMAL(5, 2) NOT NULL, - FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) -); - -CREATE TABLE monutil_urlLogs ( - logID BIGSERIAL PRIMARY KEY, - hostname VARCHAR(255) NOT NULL, - url TEXT NOT NULL, - logTime TIMESTAMP NOT NULL, - responseTime DECIMAL(5, 2) NOT NULL, - FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) -); - -CREATE TABLE monutil_ipblock ( - logID BIGSERIAL PRIMARY KEY, - hostname VARCHAR(255) NOT NULL, - blockedIPAddress VARCHAR(45) NOT NULL, - jail VARCHAR(255), - live BOOLEAN NOT NULL, - logTime TIMESTAMP NOT NULL, - FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) -); - -CREATE TABLE monutil_geoip ( - geoID BIGSERIAL PRIMARY KEY, - logID BIGINT NOT NULL, - hostname VARCHAR(255), - city VARCHAR(255), - region VARCHAR(255), - country VARCHAR(255), - loc VARCHAR(50), - org VARCHAR(255), - postal VARCHAR(20), - timezone VARCHAR(50), - FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) -);