Updated log class ready for IP block changes. Untested.
This commit is contained in:
parent
bed6ff0b94
commit
7f34688330
GPG key ID:
EB8F975CF60BCBFF
4 changed files with 243 additions and 4 deletions
145
log.py
145
log.py
|
|
@ -24,6 +24,8 @@
|
|||
|
||||
import pyodbc
|
||||
import time
|
||||
import requests
|
||||
from retrying import retry
|
||||
from odbcReferences import driver
|
||||
from config import logRetentionDays, maximumSQLAttempts, loggingMode
|
||||
from datetime import datetime, timedelta
|
||||
|
|
@ -99,6 +101,149 @@ class logsManager:
|
|||
|
||||
time.sleep(1)
|
||||
|
||||
@retry(stop_max_attempt_number=3, wait_fixed=1000)
|
||||
def geoIPLookup(self, ip, token):
|
||||
|
||||
global delayUntil
|
||||
|
||||
try:
|
||||
|
||||
if 'delayUntil' in locals() and datetime.now() < delayUntil:
|
||||
|
||||
print("Rate limit exceeded. Please wait before trying again.")
|
||||
return None
|
||||
|
||||
url = f"https://ipinfo.io/{ip}?token={token}"
|
||||
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
if response.status_code == 429:
|
||||
|
||||
print("Rate limit exceeded. Please wait before trying again.")
|
||||
today = datetime.now()
|
||||
|
||||
if today.month == 12:
|
||||
|
||||
delayUntil = datetime(today.year + 1, 1, 1)
|
||||
|
||||
else:
|
||||
|
||||
delayUntil = datetime(today.year, today.month + 1, 1)
|
||||
|
||||
return None
|
||||
|
||||
geoinfo = {
|
||||
"ip": data.get("ip"),
|
||||
"hostname": data.get("hostname"),
|
||||
"city": data.get("city"),
|
||||
"region": data.get("region"),
|
||||
"country": data.get("country"),
|
||||
"loc": data.get("loc"),
|
||||
"org": data.get("org"),
|
||||
"postal": data.get("postal"),
|
||||
"timezone": data.get("timezone"),
|
||||
}
|
||||
|
||||
|
||||
return geoinfo
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("An error occurred whilst doing a GeoIP lookup.", e)
|
||||
raise
|
||||
|
||||
def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token):
|
||||
|
||||
currentAttempts = 1
|
||||
|
||||
self.insertHost(hostname, ipAddress)
|
||||
self.deleteOldLogs("monutil_hostlogs", "logTime")
|
||||
|
||||
while currentAttempts <= maximumSQLAttempts:
|
||||
|
||||
try:
|
||||
|
||||
conn = pyodbc.connect(self.conn_str)
|
||||
cursor = conn.cursor()
|
||||
|
||||
if live == 0:
|
||||
|
||||
cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail)
|
||||
autoIncrementID = 0
|
||||
|
||||
else:
|
||||
|
||||
if loggingMode == 'mariadb':
|
||||
|
||||
cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
|
||||
cursor.execute("SELECT LAST_INSERT_ID()")
|
||||
|
||||
elif loggingMode == 'mssql':
|
||||
|
||||
cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?); SELECT LAST_INSERT_ID();", hostname, blockedIPAddress, jail, live, logTime)
|
||||
|
||||
elif loggingMode == 'postgresql':
|
||||
|
||||
cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING id;", hostname, blockedIPAddress, jail, live, logTime)
|
||||
|
||||
else:
|
||||
|
||||
raise Exception("A serious error has occurred. Unrecognised DBMS.")
|
||||
|
||||
|
||||
autoIncrementID = cursor.fetchone()[0]
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
if autoIncrementID > 0:
|
||||
|
||||
geoinfo = self.geoIPLookup(blockedIPAddress, token)
|
||||
self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"], geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"])
|
||||
|
||||
|
||||
break
|
||||
|
||||
except pyodbc.Error as ex:
|
||||
|
||||
currentAttempts += 1
|
||||
print("Error inserting/updating data in monutil_ipblock:", ex)
|
||||
|
||||
if not currentAttempts <= maximumSQLAttempts:
|
||||
raise
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone):
|
||||
|
||||
currentAttempts = 1
|
||||
|
||||
while currentAttempts <= maximumSQLAttempts:
|
||||
|
||||
try:
|
||||
conn = pyodbc.connect(self.conn_str)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID,
|
||||
hostname, city, region, country, loc, org, postal, timezone)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
break
|
||||
|
||||
except pyodbc.Error as ex:
|
||||
|
||||
currentAttempts += 1
|
||||
print("Error inserting data in monutil_geoip:", ex)
|
||||
|
||||
if not currentAttempts <= maximumSQLAttempts:
|
||||
raise
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime):
|
||||
|
||||
currentAttempts = 1
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_hostlogs (
|
||||
logID INT AUTO_INCREMENT PRIMARY KEY,
|
||||
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
cpu DECIMAL(5, 2) NOT NULL,
|
||||
|
|
@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_urlLogs (
|
||||
logID INT AUTO_INCREMENT PRIMARY KEY,
|
||||
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
responseTime DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_ipblock (
|
||||
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
blockedIPAddress VARCHAR(45) NOT NULL,
|
||||
jail VARCHAR(255),
|
||||
live BOOLEAN NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_geoip (
|
||||
geoID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
logID BIGINT NOT NULL,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
city VARCHAR(255),
|
||||
region VARCHAR(255),
|
||||
country VARCHAR(255),
|
||||
loc VARCHAR(50),
|
||||
org VARCHAR(255),
|
||||
postal VARCHAR(20),
|
||||
timezone VARCHAR(50),
|
||||
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
|
||||
);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_hostlogs (
|
||||
logID INT IDENTITY(1, 1) PRIMARY KEY,
|
||||
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
cpu DECIMAL(5, 2) NOT NULL,
|
||||
|
|
@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_urlLogs (
|
||||
logID INT IDENTITY(1, 1) PRIMARY KEY,
|
||||
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
url NVARCHAR(MAX) NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
responseTime DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_ipblock (
|
||||
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
blockedIPAddress NVARCHAR(45) NOT NULL,
|
||||
jail NVARCHAR(255),
|
||||
live BIT NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_geoip (
|
||||
geoID BIGINT IDENTITY(1,1) PRIMARY KEY,
|
||||
logID BIGINT NOT NULL,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
city NVARCHAR(255),
|
||||
region NVARCHAR(255),
|
||||
country NVARCHAR(255),
|
||||
loc NVARCHAR(50),
|
||||
org NVARCHAR(255),
|
||||
postal NVARCHAR(20),
|
||||
timezone NVARCHAR(50),
|
||||
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
|
||||
);
|
||||
|
|
|
|||
46
schema/postgresql.sql
Normal file
46
schema/postgresql.sql
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
CREATE TABLE monutil_hosts (
|
||||
hostname VARCHAR(255) PRIMARY KEY,
|
||||
ipAddress VARCHAR(15) NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_hostlogs (
|
||||
logID BIGSERIAL PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
logTime TIMESTAMP NOT NULL,
|
||||
cpu DECIMAL(5, 2) NOT NULL,
|
||||
memory DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_urlLogs (
|
||||
logID BIGSERIAL PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
logTime TIMESTAMP NOT NULL,
|
||||
responseTime DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_ipblock (
|
||||
logID BIGSERIAL PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
blockedIPAddress VARCHAR(45) NOT NULL,
|
||||
jail VARCHAR(255),
|
||||
live BOOLEAN NOT NULL,
|
||||
logTime TIMESTAMP NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_geoip (
|
||||
geoID BIGSERIAL PRIMARY KEY,
|
||||
logID BIGINT NOT NULL,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
city VARCHAR(255),
|
||||
region VARCHAR(255),
|
||||
country VARCHAR(255),
|
||||
loc VARCHAR(50),
|
||||
org VARCHAR(255),
|
||||
postal VARCHAR(20),
|
||||
timezone VARCHAR(50),
|
||||
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
|
||||
);
|
||||
Loading…
Reference in a new issue