monutil/log.py

#!/usr/bin/python3

# MIT License

# Copyright (c) 2024 Thomas Williams - https://git.server.wales/thomas

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import pyodbc
import time
import requests
from retrying import retry
from odbcReferences import driver
from config import logRetentionDaysIPBlock, logRetentionDaysHost, logRetentionDaysURL, maximumSQLAttempts, loggingMode
from datetime import datetime, timedelta

class logsManager:
    
    def __init__(self, server, database, username, password):

        self.conn_str = 'DRIVER=' + driver + ';SERVER=' + server + ';DATABASE=' + database + ';UID=' + username + ';PWD=' + password

    def insertHost(self, hostname, ipAddress):

        currentAttempts = 1

        while currentAttempts <= maximumSQLAttempts:

            try:

                conn = pyodbc.connect(self.conn_str)
                cursor = conn.cursor()
                cursor.execute("SELECT COUNT(*) FROM monutil_hosts WHERE hostname = ?", hostname)

                if cursor.fetchone()[0] == 0:

                    cursor.execute("INSERT INTO monutil_hosts (hostname, ipAddress) VALUES (?, ?)", hostname, ipAddress)
                    conn.commit()
                    conn.close()
                    break

                else:

                    cursor.execute("UPDATE monutil_hosts SET ipAddress = ? WHERE hostname = ?", ipAddress, hostname)
                    conn.close()
                    break

            except pyodbc.Error as ex:

                currentAttempts += 1
                print("SQL Error: {}".format(str(ex)))
                
                if not currentAttempts <= maximumSQLAttempts:
                    raise

                time.sleep(1)


    def insertHostLog(self, hostname, ipAddress, log_time, cpu, memory):
        
        currentAttempts = 1

        self.insertHost(hostname, ipAddress)
        self.deleteOldLogs("hostlogs", "logTime", logRetentionDaysHost)
        
        while currentAttempts <= maximumSQLAttempts:

            try:

                conn = pyodbc.connect(self.conn_str)
                cursor = conn.cursor()

                cursor.execute("INSERT INTO monutil_hostlogs (hostname, logTime, cpu, memory) VALUES (?, ?, ?, ?)", hostname, log_time, cpu, memory)
                conn.commit()
                conn.close()
                break

            except pyodbc.Error as ex:
            
                currentAttempts += 1
                print("Error inserting data: {}".format(str(ex)))

                if not currentAttempts <= maximumSQLAttempts:
                    raise

                time.sleep(1)

    @retry(stop_max_attempt_number=3, wait_fixed=1000)
    def geoIPLookup(self, ip, token):

        global delayUntil
        
        try:

            conn = pyodbc.connect(self.conn_str)
            cursor = conn.cursor()
            timeThreshold = datetime.now() - timedelta(days=1)

            cursor.execute("SELECT blockedipaddress, monutil_geoip.hostname, city, region, country, loc, org, postal, timezone FROM monutil_geoip INNER JOIN monutil_ipblock ON monutil_geoip.logID = monutil_ipblock.logID WHERE logtime >= ? AND blockedipaddress = ?", timeThreshold, ip)
            
            rows = cursor.fetchall()

            geoinfo = []

            for row in rows:
                geoinfo = {
                        "ip": row.blockedipaddress,
                        "hostname": row.hostname,
                        "city": row.city,
                        "region": row.region,
                        "country": row.country,
                        "loc": row.loc,
                        "org": row.org,
                        "postal": row.postal,
                        "timezone": row.timezone,
                        }
            
            if not geoinfo:
                
                if 'delayUntil' in locals() and datetime.now() < delayUntil:
                    
                    print("Rate limit exceeded. Please wait before trying again.")
                    return None

                url = f"https://ipinfo.io/{ip}?token={token}"

                response = requests.get(url)
                response.raise_for_status()
                data = response.json()
            
                if response.status_code == 429:

                    print("Rate limit exceeded. Please wait before trying again.")
                    today = datetime.now()
                
                    if today.month == 12:
                    
                        delayUntil = datetime(today.year + 1, 1, 1)
                
                    else:
                    
                        delayUntil = datetime(today.year, today.month + 1, 1)
                
                    return None
            
                geoinfo = {
                    "ip": data.get("ip"),
                    "hostname": data.get("hostname"),
                    "city": data.get("city"),
                    "region": data.get("region"),
                    "country": data.get("country"),
                    "loc": data.get("loc"),
                    "org": data.get("org"),
                    "postal": data.get("postal"),
                    "timezone": data.get("timezone"),
                    }

            return geoinfo

        except Exception as e:

            print("An error occurred whilst doing a GeoIP lookup.", e)
            raise

    def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token):
        
        currentAttempts = 1

        self.insertHost(hostname, ipAddress)
        self.deleteOldLogs("iplogs", "logTime", logRetentionDaysIPBlock)

        while currentAttempts <= maximumSQLAttempts:

            try:

                conn = pyodbc.connect(self.conn_str)
                cursor = conn.cursor()
 
                if int(live) == 0:
                
                    if loggingMode == 'mssql' or loggingMode == 'mariadb':
                        cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail)
                    elif loggingMode == 'postgresql':
                        cursor.execute("UPDATE monutil_ipblock SET live = FALSE WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = TRUE", hostname, blockedIPAddress, jail)
                    else:
                        raise Exception("A serious error has occurred. Unrecognised DBMS.")
                    
                    autoIncrementID = 0

                else:
                   
                    if loggingMode == 'mariadb':

                        cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
                        cursor.execute("SELECT LAST_INSERT_ID()")
                        autoIncrementID = cursor.fetchone()[0]

                    elif loggingMode == 'mssql':

                        query = cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) OUTPUT INSERTED.logID VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
                        autoIncrementID = query.fetchone()[0]

                    elif loggingMode == 'postgresql':

                        cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING logID;", hostname, blockedIPAddress, jail, live, logTime)
                        autoIncrementID = cursor.fetchone()[0]

                    else:

                        raise Exception("A serious error has occurred. Unrecognised DBMS.")

                conn.commit()
                conn.close()
                
                if autoIncrementID > 0:
                    
                    geoinfo = self.geoIPLookup(blockedIPAddress, token)
                    self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"],  geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"])


                break

            except pyodbc.Error as ex:

                currentAttempts += 1
                print("Error inserting/updating data in monutil_ipblock:", ex)

                if not currentAttempts <= maximumSQLAttempts:
                    raise

                time.sleep(1)

    def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone):
        
        currentAttempts = 1

        while currentAttempts <= maximumSQLAttempts:
            
            try:
                conn = pyodbc.connect(self.conn_str)
                cursor = conn.cursor()

                cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID, 
                               hostname, city, region, country, loc, org, postal, timezone)

                conn.commit()
                conn.close()
                break

            except pyodbc.Error as ex:
            
                currentAttempts += 1
                print("Error inserting data in monutil_geoip:", ex)

                if not currentAttempts <= maximumSQLAttempts:
                    raise

                time.sleep(1)


    def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime):

        currentAttempts = 1

        self.insertHost(hostname, ipAddress)
        self.deleteOldLogs("urllogs", "logTime", logRetentionDaysURL)
        
        while currentAttempts <= maximumSQLAttempts:

            try:

                conn = pyodbc.connect(self.conn_str)
                cursor = conn.cursor()

                cursor.execute("INSERT INTO monutil_urlLogs (hostname, url, logTime, responseTime) VALUES (?, ?, ?, ?)", hostname, url, log_time, responseTime)
                conn.commit()
                conn.close()
                break

            except pyodbc.Error as ex:

                currentAttempts += 1
                print("Error inserting data into monutil_urlLogs:", ex)

                if not currentAttempts <= maximumSQLAttempts:
                    raise

                time.sleep(1)

    def deleteOldLogs(self, logType, logTimeColumn, logRetentionDays):
        
        currentAttempts = 1

        while currentAttempts <= maximumSQLAttempts:

            try:

                if logType == 'hostlogs':
                    tableName = 'monutil_hostLogs'
                elif logType == 'urllogs':
                    tableName = 'monutil_urlLogs'
                elif logType == 'iplogs':
                    tableName = 'monutil_ipblock'

                conn = pyodbc.connect(self.conn_str)
                cursor = conn.cursor()

                if loggingMode == 'mssql':
                    
                    oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC"

                elif loggingMode == 'mariadb' or loggingMode == 'postgresql':

                    oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1"

                cursor.execute(oldestLogQuery)

                if cursor.rowcount > 0:
                    oldestLogTime = cursor.fetchone()[0]

                if cursor.rowcount > 0 and oldestLogTime < datetime.now() - timedelta(days=int(logRetentionDays)):

                    if logType == 'hostlogs':

                        deleteQuery = f"DELETE FROM {tableName} WHERE {logTimeColumn} < ?"

                        cursor.execute(deleteQuery, datetime.now() - timedelta(days=int(logRetentionDaysHost)))
                        conn.commit()

                    elif logType == 'urllogs':

                        deleteQuery = f"DELETE FROM {tableName} WHERE {logTimeColumn} < ?"

                        cursor.execute(deleteQuery, datetime.now() - timedelta(days=int(logRetentionDaysURL)))
                        conn.commit()

                    elif logType == 'iplogs':
                        
                        deleteQuery = f"DELETE FROM monutil_geoip WHERE logID IN (SELECT logID FROM monutil_ipblock WHERE {logTimeColumn} < ?)"
                        
                        cursor.execute(deleteQuery, datetime.now() - timedelta(days=int(logRetentionDaysIPBlock)))
                        conn.commit()

                        deleteQuery = f"DELETE FROM monutil_ipblock WHERE {logTimeColumn} < ?"

                        cursor.execute(deleteQuery, datetime.now() - timedelta(days=int(logRetentionDaysIPBlock)))
                        conn.commit()
                
                break

            except pyodbc.Error as ex:

                currentAttempts += 1
                print("Error deleting old logs: {}".format(str(ex)))

                if not currentAttempts <= maximumSQLAttempts:
                    raise

                time.sleep(1)