Defect Fix

log.py

@@ -24,6 +24,8 @@
 
 import pyodbc
 import time
+import requests
+from retrying import retry
 from odbcReferences import driver
 from config import logRetentionDays, maximumSQLAttempts, loggingMode
 from datetime import datetime, timedelta
@@ -99,6 +101,155 @@ class logsManager:
 
                 time.sleep(1)
 
+    @retry(stop_max_attempt_number=3, wait_fixed=1000)
+    def geoIPLookup(self, ip, token):
+
+        global delayUntil
+        
+        try:
+            
+            if 'delayUntil' in locals() and datetime.now() < delayUntil:
+                    
+                print("Rate limit exceeded. Please wait before trying again.")
+                return None
+
+            url = f"https://ipinfo.io/{ip}?token={token}"
+            
+            response = requests.get(url)
+            response.raise_for_status()
+            data = response.json()
+            
+            if response.status_code == 429:
+                
+                print("Rate limit exceeded. Please wait before trying again.")
+                today = datetime.now()
+                
+                if today.month == 12:
+                    
+                    delayUntil = datetime(today.year + 1, 1, 1)
+                
+                else:
+                    
+                    delayUntil = datetime(today.year, today.month + 1, 1)
+                
+                return None
+            
+            geoinfo = {
+                    "ip": data.get("ip"),
+                    "hostname": data.get("hostname"),
+                    "city": data.get("city"),
+                    "region": data.get("region"),
+                    "country": data.get("country"),
+                    "loc": data.get("loc"),
+                    "org": data.get("org"),
+                    "postal": data.get("postal"),
+                    "timezone": data.get("timezone"),
+                    }
+
+            
+            return geoinfo
+
+        except Exception as e:
+
+            print("An error occurred whilst doing a GeoIP lookup.", e)
+            raise
+
+    def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token):
+        
+        currentAttempts = 1
+
+        self.insertHost(hostname, ipAddress)
+        self.deleteOldLogs("monutil_hostlogs", "logTime")
+
+        while currentAttempts <= maximumSQLAttempts:
+
+            try:
+
+                conn = pyodbc.connect(self.conn_str)
+                cursor = conn.cursor()
+ 
+                if int(live) == 0:
+                
+                    if loggingMode == 'mssql' or loggingMode == 'mariadb':
+                        cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail)
+                    elif loggingMode == 'postgresql':
+                        cursor.execute("UPDATE monutil_ipblock SET live = FALSE WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = TRUE", hostname, blockedIPAddress, jail)
+                    else:
+                        raise Exception("A serious error has occurred. Unrecognised DBMS.")
+                    
+                    autoIncrementID = 0
+
+                else:
+                   
+                    if loggingMode == 'mariadb':
+
+                        cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
+                        cursor.execute("SELECT LAST_INSERT_ID()")
+                        autoIncrementID = cursor.fetchone()[0]
+
+                    elif loggingMode == 'mssql':
+
+                        query = cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) OUTPUT INSERTED.logID VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
+                        autoIncrementID = query.fetchone()[0]
+
+                    elif loggingMode == 'postgresql':
+
+                        cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING logID;", hostname, blockedIPAddress, jail, live, logTime)
+                        autoIncrementID = cursor.fetchone()[0]
+
+                    else:
+
+                        raise Exception("A serious error has occurred. Unrecognised DBMS.")
+
+                conn.commit()
+                conn.close()
+                
+                if autoIncrementID > 0:
+                    
+                    geoinfo = self.geoIPLookup(blockedIPAddress, token)
+                    self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"],  geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"])
+
+
+                break
+
+            except pyodbc.Error as ex:
+
+                currentAttempts += 1
+                print("Error inserting/updating data in monutil_ipblock:", ex)
+
+                if not currentAttempts <= maximumSQLAttempts:
+                    raise
+
+                time.sleep(1)
+
+    def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone):
+        
+        currentAttempts = 1
+
+        while currentAttempts <= maximumSQLAttempts:
+            
+            try:
+                conn = pyodbc.connect(self.conn_str)
+                cursor = conn.cursor()
+
+                cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID, 
+                               hostname, city, region, country, loc, org, postal, timezone)
+
+                conn.commit()
+                conn.close()
+                break
+
+            except pyodbc.Error as ex:
+            
+                currentAttempts += 1
+                print("Error inserting data in monutil_geoip:", ex)
+
+                if not currentAttempts <= maximumSQLAttempts:
+                    raise
+
+                time.sleep(1)
+
+
     def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime):
 
         currentAttempts = 1
@@ -143,7 +294,7 @@ class logsManager:
                     
                     oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC"
 
-                elif loggingMode == 'mariadb':
+                elif loggingMode == 'mariadb' or loggingMode == 'postgresql':
 
                     oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1"
 

odbcReferences.py

@@ -26,6 +26,7 @@ import config
 
 odbcMariaDB = '{MariaDB}'
 odbcMSSQL = '{ODBC Driver 17 for SQL Server}'
+odbcPostgreSQL = '{PostgreSQL}'
 
 # DO NOT MODIFY ANYTHING BELOW THIS LINE
 
@@ -38,3 +39,7 @@ if config.loggingMode == 'mariadb':
 elif config.loggingMode == 'mssql':
 
     driver = odbcMSSQL
+
+elif config.loggingMode == 'postgresql':
+
+    driver = odbcPostgreSQL

rabbitmq.py

@@ -88,6 +88,13 @@ class rabbitMQClient:
                 manager.insertURLLog(result[0], result[1], result[2], result[4], result[5])
                 ch.basic_ack(delivery_tag=method.delivery_tag)
 
+            if result[3] == 'ipBlock':
+                
+                result[2] = datetime.datetime.fromisoformat(result[2])
+                manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
+                manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken)
+                ch.basic_ack(delivery_tag=method.delivery_tag)
+
         context = ssl.create_default_context(
                 cafile=self.ca)
         context.verify_mode = ssl.CERT_REQUIRED

reportIPBlock.py

@@ -0,0 +1,116 @@
+def main():
+
+    import socket
+    import os
+    import config
+    import datetime
+    import signal
+    import threading
+    from log import logsManager
+    import sys
+    import select
+
+    socketFile = '/etc/monutil/ip.socket'
+
+    if len(sys.argv) == 4:
+
+        hostname = socket.gethostname()
+        ipAddress = socket.gethostbyname(hostname)
+        blockedIPAddress = sys.argv[1]
+        jail = sys.argv[2]
+        live = int(sys.argv[3])
+        logTime = datetime.datetime.now()
+
+        clientSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        clientSocket.connect(socketFile)
+        data = f"{hostname}|{ipAddress}|{logTime}|ipBlock|{blockedIPAddress}|{jail}|{live}"
+        clientSocket.send(data.encode('utf-8'))
+        clientSocket.close()
+        sys.exit(0)
+
+    else:
+        
+        if os.path.exists(socketFile):
+            os.remove(socketFile)
+
+        stopEvent = threading.Event()
+        failureEvent = threading.Event()
+        
+        dataBuffer = []
+
+        def publishData(stopEvent):
+
+            try:
+
+                if config.loggingMode == 'rabbitmq':
+
+                    import rabbitmq
+                    rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey)
+            
+                    while not (stopEvent.is_set() and not failureEvent.is_set()):
+                        
+                        if dataBuffer:
+                            
+                            data = dataBuffer.pop(0)
+                    
+                            if config.loggingMode == 'rabbitmq':
+                            
+                                rabbitmq.publish(f"{data}")
+                        
+                            else:
+                            
+                                print("Not yet implemented")
+
+            except Exception:
+                failureEvent.set()
+
+        def cleanup(signum, frame):
+            
+            print("Signal received, shutting down...")
+            stopEvent.set() 
+
+        signal.signal(signal.SIGTERM, cleanup)
+        signal.signal(signal.SIGINT, cleanup)
+
+        def server(stopEvent):
+
+            try:
+                
+                serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                serverSocket.bind(socketFile)
+                serverSocket.listen(50)
+
+                while not (stopEvent.is_set() and not failureEvent.is_set()):
+            
+                    readable, _, _ = select.select([serverSocket], [], [], 1.0)
+                
+                    if readable:
+                        clientSocket, _ = serverSocket.accept()
+                        data = clientSocket.recv(1024).decode('utf-8')
+                    
+                        if data:
+                
+                            dataBuffer.append(data)
+                            print(data)
+            
+                            clientSocket.close()
+
+            except Exception:
+                
+                failureEvent.set()
+
+        publishThread = threading.Thread(target=publishData, args=(stopEvent,))
+        serverThread = threading.Thread(target=server, args=(stopEvent,))
+        
+        publishThread.start()
+        serverThread.start()
+        
+        publishThread.join()
+        serverThread.join()
+
+        if failureEvent.is_set():
+            print("One of the threads failed. Terminating")
+            sys.exit(1)
+
+if __name__ == "__main__":
+    main()

schema/mariadb.sql

@@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
 );
 
 CREATE TABLE monutil_hostlogs (
-    logID INT AUTO_INCREMENT PRIMARY KEY,
+    logID BIGINT AUTO_INCREMENT PRIMARY KEY,
     hostname VARCHAR(255) NOT NULL,
     logTime DATETIME NOT NULL,
     cpu DECIMAL(5, 2) NOT NULL,
@@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs (
 );
 
 CREATE TABLE monutil_urlLogs (
-    logID INT AUTO_INCREMENT PRIMARY KEY,
+    logID BIGINT AUTO_INCREMENT PRIMARY KEY,
     hostname VARCHAR(255) NOT NULL,
     url TEXT NOT NULL,
     logTime DATETIME NOT NULL,
     responseTime DECIMAL(5, 2) NOT NULL,
     FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
 );
+
+CREATE TABLE monutil_ipblock (
+    logID BIGINT AUTO_INCREMENT PRIMARY KEY,
+    hostname VARCHAR(255) NOT NULL,
+    blockedIPAddress VARCHAR(45) NOT NULL,
+    jail VARCHAR(255),
+    live BOOLEAN NOT NULL,
+    logTime DATETIME NOT NULL,
+    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
+);
+
+CREATE TABLE monutil_geoip (
+    geoID BIGINT AUTO_INCREMENT PRIMARY KEY,
+    logID BIGINT NOT NULL,
+    hostname VARCHAR(255),
+    city VARCHAR(255),
+    region VARCHAR(255),
+    country VARCHAR(255),
+    loc VARCHAR(50),
+    org VARCHAR(255),
+    postal VARCHAR(20),
+    timezone VARCHAR(50),
+    FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
+);

schema/mssql.sql

@@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
 );
 
 CREATE TABLE monutil_hostlogs (
-    logID INT IDENTITY(1, 1) PRIMARY KEY,
+    logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
     hostname NVARCHAR(255) NOT NULL,
     logTime DATETIME NOT NULL,
     cpu DECIMAL(5, 2) NOT NULL,
@@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs (
 );
 
 CREATE TABLE monutil_urlLogs (
-    logID INT IDENTITY(1, 1) PRIMARY KEY,
+    logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
     hostname NVARCHAR(255) NOT NULL,
     url NVARCHAR(MAX) NOT NULL,
     logTime DATETIME NOT NULL,
     responseTime DECIMAL(5, 2) NOT NULL,
     FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
 );
+
+CREATE TABLE monutil_ipblock (
+    logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
+    hostname NVARCHAR(255) NOT NULL,
+    blockedIPAddress NVARCHAR(45) NOT NULL,
+    jail NVARCHAR(255),
+    live BIT NOT NULL,
+    logTime DATETIME NOT NULL,
+    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
+);
+
+CREATE TABLE monutil_geoip (
+    geoID BIGINT IDENTITY(1,1) PRIMARY KEY,
+    logID BIGINT NOT NULL,
+    hostname NVARCHAR(255),
+    city NVARCHAR(255),
+    region NVARCHAR(255),
+    country NVARCHAR(255),
+    loc NVARCHAR(50),
+    org NVARCHAR(255),
+    postal NVARCHAR(20),
+    timezone NVARCHAR(50),
+    FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
+);

schema/postgresql.sql

@@ -0,0 +1,46 @@
+CREATE TABLE monutil_hosts (
+    hostname VARCHAR(255) PRIMARY KEY,
+    ipAddress VARCHAR(15) NOT NULL
+);
+
+CREATE TABLE monutil_hostlogs (
+    logID BIGSERIAL PRIMARY KEY,
+    hostname VARCHAR(255) NOT NULL,
+    logTime TIMESTAMP NOT NULL,
+    cpu DECIMAL(5, 2) NOT NULL,
+    memory DECIMAL(5, 2) NOT NULL,
+    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
+);
+
+CREATE TABLE monutil_urlLogs (
+    logID BIGSERIAL PRIMARY KEY,
+    hostname VARCHAR(255) NOT NULL,
+    url TEXT NOT NULL,
+    logTime TIMESTAMP NOT NULL,
+    responseTime DECIMAL(5, 2) NOT NULL,
+    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
+);
+
+CREATE TABLE monutil_ipblock (
+    logID BIGSERIAL PRIMARY KEY,
+    hostname VARCHAR(255) NOT NULL,
+    blockedIPAddress VARCHAR(45) NOT NULL,
+    jail VARCHAR(255),
+    live BOOLEAN NOT NULL,
+    logTime TIMESTAMP NOT NULL,
+    FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
+);
+
+CREATE TABLE monutil_geoip (
+    geoID BIGSERIAL PRIMARY KEY,
+    logID BIGINT NOT NULL,
+    hostname VARCHAR(255),
+    city VARCHAR(255),
+    region VARCHAR(255),
+    country VARCHAR(255),
+    loc VARCHAR(50),
+    org VARCHAR(255),
+    postal VARCHAR(20),
+    timezone VARCHAR(50),
+    FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
+);