diff --git a/log.py b/log.py index 4778890..5546511 100644 --- a/log.py +++ b/log.py @@ -24,6 +24,8 @@ import pyodbc import time +import requests +from retrying import retry from odbcReferences import driver from config import logRetentionDays, maximumSQLAttempts, loggingMode from datetime import datetime, timedelta @@ -99,6 +101,149 @@ class logsManager: time.sleep(1) + @retry(stop_max_attempt_number=3, wait_fixed=1000) + def geoIPLookup(self, ip, token): + + global delayUntil + + try: + + if 'delayUntil' in locals() and datetime.now() < delayUntil: + + print("Rate limit exceeded. Please wait before trying again.") + return None + + url = f"https://ipinfo.io/{ip}?token={token}" + + response = requests.get(url) + response.raise_for_status() + data = response.json() + + if response.status_code == 429: + + print("Rate limit exceeded. Please wait before trying again.") + today = datetime.now() + + if today.month == 12: + + delayUntil = datetime(today.year + 1, 1, 1) + + else: + + delayUntil = datetime(today.year, today.month + 1, 1) + + return None + + geoinfo = { + "ip": data.get("ip"), + "hostname": data.get("hostname"), + "city": data.get("city"), + "region": data.get("region"), + "country": data.get("country"), + "loc": data.get("loc"), + "org": data.get("org"), + "postal": data.get("postal"), + "timezone": data.get("timezone"), + } + + + return geoinfo + + except Exception as e: + + print("An error occurred whilst doing a GeoIP lookup.", e) + raise + + def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token): + + currentAttempts = 1 + + self.insertHost(hostname, ipAddress) + self.deleteOldLogs("monutil_hostlogs", "logTime") + + while currentAttempts <= maximumSQLAttempts: + + try: + + conn = pyodbc.connect(self.conn_str) + cursor = conn.cursor() + + if live == 0: + + cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail) + autoIncrementID = 0 + + else: + + if loggingMode == 'mariadb': + + cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime) + cursor.execute("SELECT LAST_INSERT_ID()") + + elif loggingMode == 'mssql': + + cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?); SELECT LAST_INSERT_ID();", hostname, blockedIPAddress, jail, live, logTime) + + elif loggingMode == 'postgresql': + + cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING id;", hostname, blockedIPAddress, jail, live, logTime) + + else: + + raise Exception("A serious error has occurred. Unrecognised DBMS.") + + + autoIncrementID = cursor.fetchone()[0] + + conn.commit() + conn.close() + + if autoIncrementID > 0: + + geoinfo = self.geoIPLookup(blockedIPAddress, token) + self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"], geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"]) + + + break + + except pyodbc.Error as ex: + + currentAttempts += 1 + print("Error inserting/updating data in monutil_ipblock:", ex) + + if not currentAttempts <= maximumSQLAttempts: + raise + + time.sleep(1) + + def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone): + + currentAttempts = 1 + + while currentAttempts <= maximumSQLAttempts: + + try: + conn = pyodbc.connect(self.conn_str) + cursor = conn.cursor() + + cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID, + hostname, city, region, country, loc, org, postal, timezone) + + conn.commit() + conn.close() + break + + except pyodbc.Error as ex: + + currentAttempts += 1 + print("Error inserting data in monutil_geoip:", ex) + + if not currentAttempts <= maximumSQLAttempts: + raise + + time.sleep(1) + + def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime): currentAttempts = 1 diff --git a/schema/mariadb.sql b/schema/mariadb.sql index 77ae561..25f006c 100644 --- a/schema/mariadb.sql +++ b/schema/mariadb.sql @@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts ( ); CREATE TABLE monutil_hostlogs ( - logID INT AUTO_INCREMENT PRIMARY KEY, + logID BIGINT AUTO_INCREMENT PRIMARY KEY, hostname VARCHAR(255) NOT NULL, logTime DATETIME NOT NULL, cpu DECIMAL(5, 2) NOT NULL, @@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs ( ); CREATE TABLE monutil_urlLogs ( - logID INT AUTO_INCREMENT PRIMARY KEY, + logID BIGINT AUTO_INCREMENT PRIMARY KEY, hostname VARCHAR(255) NOT NULL, url TEXT NOT NULL, logTime DATETIME NOT NULL, responseTime DECIMAL(5, 2) NOT NULL, FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) ); + +CREATE TABLE monutil_ipblock ( + logID BIGINT AUTO_INCREMENT PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + blockedIPAddress VARCHAR(45) NOT NULL, + jail VARCHAR(255), + live BOOLEAN NOT NULL, + logTime DATETIME NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_geoip ( + geoID BIGINT AUTO_INCREMENT PRIMARY KEY, + logID BIGINT NOT NULL, + hostname VARCHAR(255) NOT NULL, + city VARCHAR(255), + region VARCHAR(255), + country VARCHAR(255), + loc VARCHAR(50), + org VARCHAR(255), + postal VARCHAR(20), + timezone VARCHAR(50), + FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) +); diff --git a/schema/mssql.sql b/schema/mssql.sql index 1ce2843..b49676c 100644 --- a/schema/mssql.sql +++ b/schema/mssql.sql @@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts ( ); CREATE TABLE monutil_hostlogs ( - logID INT IDENTITY(1, 1) PRIMARY KEY, + logID BIGINT IDENTITY(1, 1) PRIMARY KEY, hostname NVARCHAR(255) NOT NULL, logTime DATETIME NOT NULL, cpu DECIMAL(5, 2) NOT NULL, @@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs ( ); CREATE TABLE monutil_urlLogs ( - logID INT IDENTITY(1, 1) PRIMARY KEY, + logID BIGINT IDENTITY(1, 1) PRIMARY KEY, hostname NVARCHAR(255) NOT NULL, url NVARCHAR(MAX) NOT NULL, logTime DATETIME NOT NULL, responseTime DECIMAL(5, 2) NOT NULL, FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) ); + +CREATE TABLE monutil_ipblock ( + logID BIGINT IDENTITY(1, 1) PRIMARY KEY, + hostname NVARCHAR(255) NOT NULL, + blockedIPAddress NVARCHAR(45) NOT NULL, + jail NVARCHAR(255), + live BIT NOT NULL, + logTime DATETIME NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_geoip ( + geoID BIGINT IDENTITY(1,1) PRIMARY KEY, + logID BIGINT NOT NULL, + hostname NVARCHAR(255) NOT NULL, + city NVARCHAR(255), + region NVARCHAR(255), + country NVARCHAR(255), + loc NVARCHAR(50), + org NVARCHAR(255), + postal NVARCHAR(20), + timezone NVARCHAR(50), + FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) +); diff --git a/schema/postgresql.sql b/schema/postgresql.sql new file mode 100644 index 0000000..581aa51 --- /dev/null +++ b/schema/postgresql.sql @@ -0,0 +1,46 @@ +CREATE TABLE monutil_hosts ( + hostname VARCHAR(255) PRIMARY KEY, + ipAddress VARCHAR(15) NOT NULL +); + +CREATE TABLE monutil_hostlogs ( + logID BIGSERIAL PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + logTime TIMESTAMP NOT NULL, + cpu DECIMAL(5, 2) NOT NULL, + memory DECIMAL(5, 2) NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_urlLogs ( + logID BIGSERIAL PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + url TEXT NOT NULL, + logTime TIMESTAMP NOT NULL, + responseTime DECIMAL(5, 2) NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_ipblock ( + logID BIGSERIAL PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + blockedIPAddress VARCHAR(45) NOT NULL, + jail VARCHAR(255), + live BOOLEAN NOT NULL, + logTime TIMESTAMP NOT NULL, + FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) +); + +CREATE TABLE monutil_geoip ( + geoID BIGSERIAL PRIMARY KEY, + logID BIGINT NOT NULL, + hostname VARCHAR(255) NOT NULL, + city VARCHAR(255), + region VARCHAR(255), + country VARCHAR(255), + loc VARCHAR(50), + org VARCHAR(255), + postal VARCHAR(20), + timezone VARCHAR(50), + FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID) +);