Compare commits
12 commits
bed6ff0b94
...
ef294aa1c5
Author | SHA1 | Date | |
---|---|---|---|
ef294aa1c5 | |||
e92d76cafd | |||
bf82596b00 | |||
b599a4a483 | |||
c11943cc67 | |||
daa88b46a9 | |||
387c0daab0 | |||
8ce56e19be | |||
2e04b5eeb8 | |||
46d895d900 | |||
20a2a2be26 | |||
7f34688330 |
7 changed files with 380 additions and 7 deletions
153
log.py
153
log.py
|
@ -24,6 +24,8 @@
|
|||
|
||||
import pyodbc
|
||||
import time
|
||||
import requests
|
||||
from retrying import retry
|
||||
from odbcReferences import driver
|
||||
from config import logRetentionDays, maximumSQLAttempts, loggingMode
|
||||
from datetime import datetime, timedelta
|
||||
|
@ -99,6 +101,155 @@ class logsManager:
|
|||
|
||||
time.sleep(1)
|
||||
|
||||
@retry(stop_max_attempt_number=3, wait_fixed=1000)
|
||||
def geoIPLookup(self, ip, token):
|
||||
|
||||
global delayUntil
|
||||
|
||||
try:
|
||||
|
||||
if 'delayUntil' in locals() and datetime.now() < delayUntil:
|
||||
|
||||
print("Rate limit exceeded. Please wait before trying again.")
|
||||
return None
|
||||
|
||||
url = f"https://ipinfo.io/{ip}?token={token}"
|
||||
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
if response.status_code == 429:
|
||||
|
||||
print("Rate limit exceeded. Please wait before trying again.")
|
||||
today = datetime.now()
|
||||
|
||||
if today.month == 12:
|
||||
|
||||
delayUntil = datetime(today.year + 1, 1, 1)
|
||||
|
||||
else:
|
||||
|
||||
delayUntil = datetime(today.year, today.month + 1, 1)
|
||||
|
||||
return None
|
||||
|
||||
geoinfo = {
|
||||
"ip": data.get("ip"),
|
||||
"hostname": data.get("hostname"),
|
||||
"city": data.get("city"),
|
||||
"region": data.get("region"),
|
||||
"country": data.get("country"),
|
||||
"loc": data.get("loc"),
|
||||
"org": data.get("org"),
|
||||
"postal": data.get("postal"),
|
||||
"timezone": data.get("timezone"),
|
||||
}
|
||||
|
||||
|
||||
return geoinfo
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("An error occurred whilst doing a GeoIP lookup.", e)
|
||||
raise
|
||||
|
||||
def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token):
|
||||
|
||||
currentAttempts = 1
|
||||
|
||||
self.insertHost(hostname, ipAddress)
|
||||
self.deleteOldLogs("monutil_hostlogs", "logTime")
|
||||
|
||||
while currentAttempts <= maximumSQLAttempts:
|
||||
|
||||
try:
|
||||
|
||||
conn = pyodbc.connect(self.conn_str)
|
||||
cursor = conn.cursor()
|
||||
|
||||
if int(live) == 0:
|
||||
|
||||
if loggingMode == 'mssql' or loggingMode == 'mariadb':
|
||||
cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail)
|
||||
elif loggingMode == 'postgresql':
|
||||
cursor.execute("UPDATE monutil_ipblock SET live = FALSE WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = TRUE", hostname, blockedIPAddress, jail)
|
||||
else:
|
||||
raise Exception("A serious error has occurred. Unrecognised DBMS.")
|
||||
|
||||
autoIncrementID = 0
|
||||
|
||||
else:
|
||||
|
||||
if loggingMode == 'mariadb':
|
||||
|
||||
cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
|
||||
cursor.execute("SELECT LAST_INSERT_ID()")
|
||||
autoIncrementID = cursor.fetchone()[0]
|
||||
|
||||
elif loggingMode == 'mssql':
|
||||
|
||||
query = cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) OUTPUT INSERTED.logID VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
|
||||
autoIncrementID = query.fetchone()[0]
|
||||
|
||||
elif loggingMode == 'postgresql':
|
||||
|
||||
cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING logID;", hostname, blockedIPAddress, jail, live, logTime)
|
||||
autoIncrementID = cursor.fetchone()[0]
|
||||
|
||||
else:
|
||||
|
||||
raise Exception("A serious error has occurred. Unrecognised DBMS.")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
if autoIncrementID > 0:
|
||||
|
||||
geoinfo = self.geoIPLookup(blockedIPAddress, token)
|
||||
self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"], geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"])
|
||||
|
||||
|
||||
break
|
||||
|
||||
except pyodbc.Error as ex:
|
||||
|
||||
currentAttempts += 1
|
||||
print("Error inserting/updating data in monutil_ipblock:", ex)
|
||||
|
||||
if not currentAttempts <= maximumSQLAttempts:
|
||||
raise
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone):
|
||||
|
||||
currentAttempts = 1
|
||||
|
||||
while currentAttempts <= maximumSQLAttempts:
|
||||
|
||||
try:
|
||||
conn = pyodbc.connect(self.conn_str)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID,
|
||||
hostname, city, region, country, loc, org, postal, timezone)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
break
|
||||
|
||||
except pyodbc.Error as ex:
|
||||
|
||||
currentAttempts += 1
|
||||
print("Error inserting data in monutil_geoip:", ex)
|
||||
|
||||
if not currentAttempts <= maximumSQLAttempts:
|
||||
raise
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime):
|
||||
|
||||
currentAttempts = 1
|
||||
|
@ -143,7 +294,7 @@ class logsManager:
|
|||
|
||||
oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC"
|
||||
|
||||
elif loggingMode == 'mariadb':
|
||||
elif loggingMode == 'mariadb' or loggingMode == 'postgresql':
|
||||
|
||||
oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1"
|
||||
|
||||
|
|
|
@ -26,6 +26,7 @@ import config
|
|||
|
||||
odbcMariaDB = '{MariaDB}'
|
||||
odbcMSSQL = '{ODBC Driver 17 for SQL Server}'
|
||||
odbcPostgreSQL = '{PostgreSQL}'
|
||||
|
||||
# DO NOT MODIFY ANYTHING BELOW THIS LINE
|
||||
|
||||
|
@ -38,3 +39,7 @@ if config.loggingMode == 'mariadb':
|
|||
elif config.loggingMode == 'mssql':
|
||||
|
||||
driver = odbcMSSQL
|
||||
|
||||
elif config.loggingMode == 'postgresql':
|
||||
|
||||
driver = odbcPostgreSQL
|
||||
|
|
11
rabbitmq.py
11
rabbitmq.py
|
@ -75,19 +75,26 @@ class rabbitMQClient:
|
|||
result = body.split('|')
|
||||
|
||||
if result[3] == 'cpumem':
|
||||
|
||||
|
||||
result[2] = datetime.datetime.fromisoformat(result[2])
|
||||
manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
|
||||
manager.insertHostLog(result[0], result[1], result[2], result[4], result[5])
|
||||
ch.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
if result[3] == 'url':
|
||||
|
||||
|
||||
result[2] = datetime.datetime.fromisoformat(result[2])
|
||||
manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
|
||||
manager.insertURLLog(result[0], result[1], result[2], result[4], result[5])
|
||||
ch.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
if result[3] == 'ipBlock':
|
||||
|
||||
result[2] = datetime.datetime.fromisoformat(result[2])
|
||||
manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
|
||||
manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken)
|
||||
ch.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
context = ssl.create_default_context(
|
||||
cafile=self.ca)
|
||||
context.verify_mode = ssl.CERT_REQUIRED
|
||||
|
|
116
reportIPBlock.py
Normal file
116
reportIPBlock.py
Normal file
|
@ -0,0 +1,116 @@
|
|||
def main():
|
||||
|
||||
import socket
|
||||
import os
|
||||
import config
|
||||
import datetime
|
||||
import signal
|
||||
import threading
|
||||
from log import logsManager
|
||||
import sys
|
||||
import select
|
||||
|
||||
socketFile = '/etc/monutil/ip.socket'
|
||||
|
||||
if len(sys.argv) == 4:
|
||||
|
||||
hostname = socket.gethostname()
|
||||
ipAddress = socket.gethostbyname(hostname)
|
||||
blockedIPAddress = sys.argv[1]
|
||||
jail = sys.argv[2]
|
||||
live = int(sys.argv[3])
|
||||
logTime = datetime.datetime.now()
|
||||
|
||||
clientSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
clientSocket.connect(socketFile)
|
||||
data = f"{hostname}|{ipAddress}|{logTime}|ipBlock|{blockedIPAddress}|{jail}|{live}"
|
||||
clientSocket.send(data.encode('utf-8'))
|
||||
clientSocket.close()
|
||||
sys.exit(0)
|
||||
|
||||
else:
|
||||
|
||||
if os.path.exists(socketFile):
|
||||
os.remove(socketFile)
|
||||
|
||||
stopEvent = threading.Event()
|
||||
failureEvent = threading.Event()
|
||||
|
||||
dataBuffer = []
|
||||
|
||||
def publishData(stopEvent):
|
||||
|
||||
try:
|
||||
|
||||
if config.loggingMode == 'rabbitmq':
|
||||
|
||||
import rabbitmq
|
||||
rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey)
|
||||
|
||||
while not (stopEvent.is_set() and not failureEvent.is_set()):
|
||||
|
||||
if dataBuffer:
|
||||
|
||||
data = dataBuffer.pop(0)
|
||||
|
||||
if config.loggingMode == 'rabbitmq':
|
||||
|
||||
rabbitmq.publish(f"{data}")
|
||||
|
||||
else:
|
||||
|
||||
print("Not yet implemented")
|
||||
|
||||
except Exception:
|
||||
failureEvent.set()
|
||||
|
||||
def cleanup(signum, frame):
|
||||
|
||||
print("Signal received, shutting down...")
|
||||
stopEvent.set()
|
||||
|
||||
signal.signal(signal.SIGTERM, cleanup)
|
||||
signal.signal(signal.SIGINT, cleanup)
|
||||
|
||||
def server(stopEvent):
|
||||
|
||||
try:
|
||||
|
||||
serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
serverSocket.bind(socketFile)
|
||||
serverSocket.listen(50)
|
||||
|
||||
while not (stopEvent.is_set() and not failureEvent.is_set()):
|
||||
|
||||
readable, _, _ = select.select([serverSocket], [], [], 1.0)
|
||||
|
||||
if readable:
|
||||
clientSocket, _ = serverSocket.accept()
|
||||
data = clientSocket.recv(1024).decode('utf-8')
|
||||
|
||||
if data:
|
||||
|
||||
dataBuffer.append(data)
|
||||
print(data)
|
||||
|
||||
clientSocket.close()
|
||||
|
||||
except Exception:
|
||||
|
||||
failureEvent.set()
|
||||
|
||||
publishThread = threading.Thread(target=publishData, args=(stopEvent,))
|
||||
serverThread = threading.Thread(target=server, args=(stopEvent,))
|
||||
|
||||
publishThread.start()
|
||||
serverThread.start()
|
||||
|
||||
publishThread.join()
|
||||
serverThread.join()
|
||||
|
||||
if failureEvent.is_set():
|
||||
print("One of the threads failed. Terminating")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
|
@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_hostlogs (
|
||||
logID INT AUTO_INCREMENT PRIMARY KEY,
|
||||
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
cpu DECIMAL(5, 2) NOT NULL,
|
||||
|
@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_urlLogs (
|
||||
logID INT AUTO_INCREMENT PRIMARY KEY,
|
||||
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
responseTime DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_ipblock (
|
||||
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
blockedIPAddress VARCHAR(45) NOT NULL,
|
||||
jail VARCHAR(255),
|
||||
live BOOLEAN NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_geoip (
|
||||
geoID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
logID BIGINT NOT NULL,
|
||||
hostname VARCHAR(255),
|
||||
city VARCHAR(255),
|
||||
region VARCHAR(255),
|
||||
country VARCHAR(255),
|
||||
loc VARCHAR(50),
|
||||
org VARCHAR(255),
|
||||
postal VARCHAR(20),
|
||||
timezone VARCHAR(50),
|
||||
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
|
||||
);
|
||||
|
|
|
@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_hostlogs (
|
||||
logID INT IDENTITY(1, 1) PRIMARY KEY,
|
||||
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
cpu DECIMAL(5, 2) NOT NULL,
|
||||
|
@ -13,10 +13,34 @@ CREATE TABLE monutil_hostlogs (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_urlLogs (
|
||||
logID INT IDENTITY(1, 1) PRIMARY KEY,
|
||||
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
url NVARCHAR(MAX) NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
responseTime DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_ipblock (
|
||||
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
blockedIPAddress NVARCHAR(45) NOT NULL,
|
||||
jail NVARCHAR(255),
|
||||
live BIT NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_geoip (
|
||||
geoID BIGINT IDENTITY(1,1) PRIMARY KEY,
|
||||
logID BIGINT NOT NULL,
|
||||
hostname NVARCHAR(255),
|
||||
city NVARCHAR(255),
|
||||
region NVARCHAR(255),
|
||||
country NVARCHAR(255),
|
||||
loc NVARCHAR(50),
|
||||
org NVARCHAR(255),
|
||||
postal NVARCHAR(20),
|
||||
timezone NVARCHAR(50),
|
||||
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
|
||||
);
|
||||
|
|
46
schema/postgresql.sql
Normal file
46
schema/postgresql.sql
Normal file
|
@ -0,0 +1,46 @@
|
|||
CREATE TABLE monutil_hosts (
|
||||
hostname VARCHAR(255) PRIMARY KEY,
|
||||
ipAddress VARCHAR(15) NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_hostlogs (
|
||||
logID BIGSERIAL PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
logTime TIMESTAMP NOT NULL,
|
||||
cpu DECIMAL(5, 2) NOT NULL,
|
||||
memory DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_urlLogs (
|
||||
logID BIGSERIAL PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
logTime TIMESTAMP NOT NULL,
|
||||
responseTime DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_ipblock (
|
||||
logID BIGSERIAL PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
blockedIPAddress VARCHAR(45) NOT NULL,
|
||||
jail VARCHAR(255),
|
||||
live BOOLEAN NOT NULL,
|
||||
logTime TIMESTAMP NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_geoip (
|
||||
geoID BIGSERIAL PRIMARY KEY,
|
||||
logID BIGINT NOT NULL,
|
||||
hostname VARCHAR(255),
|
||||
city VARCHAR(255),
|
||||
region VARCHAR(255),
|
||||
country VARCHAR(255),
|
||||
loc VARCHAR(50),
|
||||
org VARCHAR(255),
|
||||
postal VARCHAR(20),
|
||||
timezone VARCHAR(50),
|
||||
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
|
||||
);
|
Loading…
Reference in a new issue