log.py

@@ -24,8 +24,6 @@
 
 import pyodbc
 import time
-import requests
-from retrying import retry
 from odbcReferences import driver
 from config import logRetentionDays, maximumSQLAttempts, loggingMode
 from datetime import datetime, timedelta
@@ -101,155 +99,6 @@ class logsManager:
 
                 time.sleep(1)
 
-    @retry(stop_max_attempt_number=3, wait_fixed=1000)
-    def geoIPLookup(self, ip, token):
-
-        global delayUntil
-        
-        try:
-            
-            if 'delayUntil' in locals() and datetime.now() < delayUntil:
-                    
-                print("Rate limit exceeded. Please wait before trying again.")
-                return None
-
-            url = f"https://ipinfo.io/{ip}?token={token}"
-            
-            response = requests.get(url)
-            response.raise_for_status()
-            data = response.json()
-            
-            if response.status_code == 429:
-                
-                print("Rate limit exceeded. Please wait before trying again.")
-                today = datetime.now()
-                
-                if today.month == 12:
-                    
-                    delayUntil = datetime(today.year + 1, 1, 1)
-                
-                else:
-                    
-                    delayUntil = datetime(today.year, today.month + 1, 1)
-                
-                return None
-            
-            geoinfo = {
-                    "ip": data.get("ip"),
-                    "hostname": data.get("hostname"),
-                    "city": data.get("city"),
-                    "region": data.get("region"),
-                    "country": data.get("country"),
-                    "loc": data.get("loc"),
-                    "org": data.get("org"),
-                    "postal": data.get("postal"),
-                    "timezone": data.get("timezone"),
-                    }
-
-            
-            return geoinfo
-
-        except Exception as e:
-
-            print("An error occurred whilst doing a GeoIP lookup.", e)
-            raise
-
-    def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token):
-        
-        currentAttempts = 1
-
-        self.insertHost(hostname, ipAddress)
-        self.deleteOldLogs("monutil_hostlogs", "logTime")
-
-        while currentAttempts <= maximumSQLAttempts:
-
-            try:
-
-                conn = pyodbc.connect(self.conn_str)
-                cursor = conn.cursor()
- 
-                if int(live) == 0:
-                
-                    if loggingMode == 'mssql' or loggingMode == 'mariadb':
-                        cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail)
-                    elif loggingMode == 'postgresql':
-                        cursor.execute("UPDATE monutil_ipblock SET live = FALSE WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = TRUE", hostname, blockedIPAddress, jail)
-                    else:
-                        raise Exception("A serious error has occurred. Unrecognised DBMS.")
-                    
-                    autoIncrementID = 0
-
-                else:
-                   
-                    if loggingMode == 'mariadb':
-
-                        cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
-                        cursor.execute("SELECT LAST_INSERT_ID()")
-                        autoIncrementID = cursor.fetchone()[0]
-
-                    elif loggingMode == 'mssql':
-
-                        query = cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) OUTPUT INSERTED.logID VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
-                        autoIncrementID = query.fetchone()[0]
-
-                    elif loggingMode == 'postgresql':
-
-                        cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING logID;", hostname, blockedIPAddress, jail, live, logTime)
-                        autoIncrementID = cursor.fetchone()[0]
-
-                    else:
-
-                        raise Exception("A serious error has occurred. Unrecognised DBMS.")
-
-                conn.commit()
-                conn.close()
-                
-                if autoIncrementID > 0:
-                    
-                    geoinfo = self.geoIPLookup(blockedIPAddress, token)
-                    self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"],  geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"])
-
-
-                break
-
-            except pyodbc.Error as ex:
-
-                currentAttempts += 1
-                print("Error inserting/updating data in monutil_ipblock:", ex)
-
-                if not currentAttempts <= maximumSQLAttempts:
-                    raise
-
-                time.sleep(1)
-
-    def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone):
-        
-        currentAttempts = 1
-
-        while currentAttempts <= maximumSQLAttempts:
-            
-            try:
-                conn = pyodbc.connect(self.conn_str)
-                cursor = conn.cursor()
-
-                cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID, 
-                               hostname, city, region, country, loc, org, postal, timezone)
-
-                conn.commit()
-                conn.close()
-                break
-
-            except pyodbc.Error as ex:
-            
-                currentAttempts += 1
-                print("Error inserting data in monutil_geoip:", ex)
-
-                if not currentAttempts <= maximumSQLAttempts:
-                    raise
-
-                time.sleep(1)
-
-
     def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime):
 
         currentAttempts = 1
@@ -294,7 +143,7 @@ class logsManager:
                     
                     oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC"
 
-                elif loggingMode == 'mariadb' or loggingMode == 'postgresql':
+                elif loggingMode == 'mariadb':
 
                     oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1"
 

odbcReferences.py

@@ -26,7 +26,6 @@ import config
 
 odbcMariaDB = '{MariaDB}'
 odbcMSSQL = '{ODBC Driver 17 for SQL Server}'
-odbcPostgreSQL = '{PostgreSQL}'
 
 # DO NOT MODIFY ANYTHING BELOW THIS LINE
 
@@ -39,7 +38,3 @@ if config.loggingMode == 'mariadb':
 elif config.loggingMode == 'mssql':
 
     driver = odbcMSSQL
-
-elif config.loggingMode == 'postgresql':
-
-    driver = odbcPostgreSQL

rabbitmq.py

@@ -75,26 +75,19 @@ class rabbitMQClient:
             result = body.split('|')
             
             if result[3] == 'cpumem':
-                
+
                 result[2] = datetime.datetime.fromisoformat(result[2])
                 manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
                 manager.insertHostLog(result[0], result[1], result[2], result[4], result[5])
                 ch.basic_ack(delivery_tag=method.delivery_tag)
 
             if result[3] == 'url':
-                
+
                 result[2] = datetime.datetime.fromisoformat(result[2])
                 manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
                 manager.insertURLLog(result[0], result[1], result[2], result[4], result[5])
                 ch.basic_ack(delivery_tag=method.delivery_tag)
 
-            if result[3] == 'ipBlock':
-                
-                result[2] = datetime.datetime.fromisoformat(result[2])
-                manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
-                manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken)
-                ch.basic_ack(delivery_tag=method.delivery_tag)
-
         context = ssl.create_default_context(
                 cafile=self.ca)
         context.verify_mode = ssl.CERT_REQUIRED

reportIPBlock.py

@@ -1,116 +0,0 @@
-def main():
-
-    import socket
-    import os
-    import config
-    import datetime
-    import signal
-    import threading
-    from log import logsManager
-    import sys
-    import select
-
-    socketFile = '/etc/monutil/ip.socket'
-
-    if len(sys.argv) == 4:
-
-        hostname = socket.gethostname()
-        ipAddress = socket.gethostbyname(hostname)
-        blockedIPAddress = sys.argv[1]
-        jail = sys.argv[2]
-        live = int(sys.argv[3])
-        logTime = datetime.datetime.now()
-
-        clientSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        clientSocket.connect(socketFile)
-        data = f"{hostname}|{ipAddress}|{logTime}|ipBlock|{blockedIPAddress}|{jail}|{live}"
-        clientSocket.send(data.encode('utf-8'))
-        clientSocket.close()
-        sys.exit(0)
-
-    else:
-        
-        if os.path.exists(socketFile):
-            os.remove(socketFile)
-
-        stopEvent = threading.Event()
-        failureEvent = threading.Event()
-        
-        dataBuffer = []
-
-        def publishData(stopEvent):
-
-            try:
-
-                if config.loggingMode == 'rabbitmq':
-
-                    import rabbitmq
-                    rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey)
-            
-                    while not (stopEvent.is_set() and not failureEvent.is_set()):
-                        
-                        if dataBuffer:
-                            
-                            data = dataBuffer.pop(0)
-                    
-                            if config.loggingMode == 'rabbitmq':
-                            
-                                rabbitmq.publish(f"{data}")
-                        
-                            else:
-                            
-                                print("Not yet implemented")
-
-            except Exception:
-                failureEvent.set()
-
-        def cleanup(signum, frame):
-            
-            print("Signal received, shutting down...")
-            stopEvent.set() 
-
-        signal.signal(signal.SIGTERM, cleanup)
-        signal.signal(signal.SIGINT, cleanup)
-
-        def server(stopEvent):
-
-            try:
-                
-                serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-                serverSocket.bind(socketFile)
-                serverSocket.listen(50)
-
-                while not (stopEvent.is_set() and not failureEvent.is_set()):
-            
-                    readable, _, _ = select.select([serverSocket], [], [], 1.0)
-                
-                    if readable:
-                        clientSocket, _ = serverSocket.accept()
-                        data = clientSocket.recv(1024).decode('utf-8')
-                    
-                        if data:
-                
-                            dataBuffer.append(data)
-                            print(data)
-            
-                            clientSocket.close()
-
-            except Exception:
-                
-                failureEvent.set()
-
-        publishThread = threading.Thread(target=publishData, args=(stopEvent,))
-        serverThread = threading.Thread(target=server, args=(stopEvent,))
-        
-        publishThread.start()
-        serverThread.start()
-        
-        publishThread.join()
-        serverThread.join()
-
-        if failureEvent.is_set():
-            print("One of the threads failed. Terminating")
-            sys.exit(1)
-
-if __name__ == "__main__":
-    main()

schema/mariadb.sql

@@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
 );
 
 CREATE TABLE monutil_hostlogs (
-    logID BIGINT AUTO_INCREMENT PRIMARY KEY,
+    logID INT AUTO_INCREMENT PRIMARY KEY,
     hostname VARCHAR(255) NOT NULL,
     logTime DATETIME NOT NULL,
     cpu DECIMAL(5, 2) NOT NULL,
@@ -13,34 +13,10 @@ CREATE TABLE monutil_hostlogs (
 );
 
 CREATE TABLE monutil_urlLogs (
-    logID BIGINT AUTO_INCREMENT PRIMARY KEY,
+    logID INT AUTO_INCREMENT PRIMARY KEY,
     hostname VARCHAR(255) NOT NULL,
     url TEXT NOT NULL,
     logTime DATETIME NOT NULL,
     responseTime DECIMAL(5, 2) NOT NULL,
     FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
 );
-
-CREATE TABLE monutil_ipblock (
-    logID BIGINT AUTO_INCREMENT PRIMARY KEY,
-    hostname VARCHAR(255) NOT NULL,
-    blockedIPAddress VARCHAR(45) NOT NULL,
-    jail VARCHAR(255),
-    live BOOLEAN NOT NULL,
-    logTime DATETIME NOT NULL,
-    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
-);
-
-CREATE TABLE monutil_geoip (
-    geoID BIGINT AUTO_INCREMENT PRIMARY KEY,
-    logID BIGINT NOT NULL,
-    hostname VARCHAR(255),
-    city VARCHAR(255),
-    region VARCHAR(255),
-    country VARCHAR(255),
-    loc VARCHAR(50),
-    org VARCHAR(255),
-    postal VARCHAR(20),
-    timezone VARCHAR(50),
-    FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
-);

schema/mssql.sql

@@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
 );
 
 CREATE TABLE monutil_hostlogs (
-    logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
+    logID INT IDENTITY(1, 1) PRIMARY KEY,
     hostname NVARCHAR(255) NOT NULL,
     logTime DATETIME NOT NULL,
     cpu DECIMAL(5, 2) NOT NULL,
@@ -13,34 +13,10 @@ CREATE TABLE monutil_hostlogs (
 );
 
 CREATE TABLE monutil_urlLogs (
-    logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
+    logID INT IDENTITY(1, 1) PRIMARY KEY,
     hostname NVARCHAR(255) NOT NULL,
     url NVARCHAR(MAX) NOT NULL,
     logTime DATETIME NOT NULL,
     responseTime DECIMAL(5, 2) NOT NULL,
     FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
 );
-
-CREATE TABLE monutil_ipblock (
-    logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
-    hostname NVARCHAR(255) NOT NULL,
-    blockedIPAddress NVARCHAR(45) NOT NULL,
-    jail NVARCHAR(255),
-    live BIT NOT NULL,
-    logTime DATETIME NOT NULL,
-    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
-);
-
-CREATE TABLE monutil_geoip (
-    geoID BIGINT IDENTITY(1,1) PRIMARY KEY,
-    logID BIGINT NOT NULL,
-    hostname NVARCHAR(255),
-    city NVARCHAR(255),
-    region NVARCHAR(255),
-    country NVARCHAR(255),
-    loc NVARCHAR(50),
-    org NVARCHAR(255),
-    postal NVARCHAR(20),
-    timezone NVARCHAR(50),
-    FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
-);

schema/postgresql.sql

@@ -1,46 +0,0 @@
-CREATE TABLE monutil_hosts (
-    hostname VARCHAR(255) PRIMARY KEY,
-    ipAddress VARCHAR(15) NOT NULL
-);
-
-CREATE TABLE monutil_hostlogs (
-    logID BIGSERIAL PRIMARY KEY,
-    hostname VARCHAR(255) NOT NULL,
-    logTime TIMESTAMP NOT NULL,
-    cpu DECIMAL(5, 2) NOT NULL,
-    memory DECIMAL(5, 2) NOT NULL,
-    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
-);
-
-CREATE TABLE monutil_urlLogs (
-    logID BIGSERIAL PRIMARY KEY,
-    hostname VARCHAR(255) NOT NULL,
-    url TEXT NOT NULL,
-    logTime TIMESTAMP NOT NULL,
-    responseTime DECIMAL(5, 2) NOT NULL,
-    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
-);
-
-CREATE TABLE monutil_ipblock (
-    logID BIGSERIAL PRIMARY KEY,
-    hostname VARCHAR(255) NOT NULL,
-    blockedIPAddress VARCHAR(45) NOT NULL,
-    jail VARCHAR(255),
-    live BOOLEAN NOT NULL,
-    logTime TIMESTAMP NOT NULL,
-    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
-);
-
-CREATE TABLE monutil_geoip (
-    geoID BIGSERIAL PRIMARY KEY,
-    logID BIGINT NOT NULL,
-    hostname VARCHAR(255),
-    city VARCHAR(255),
-    region VARCHAR(255),
-    country VARCHAR(255),
-    loc VARCHAR(50),
-    org VARCHAR(255),
-    postal VARCHAR(20),
-    timezone VARCHAR(50),
-    FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
-);