Compare commits

..

No commits in common. "ef294aa1c5d41f900790ba807cfcf545061d2d4c" and "bed6ff0b94d54f971e158c8c93803cc7cbd8fe13" have entirely different histories.

7 changed files with 7 additions and 380 deletions

153
log.py
View file

@ -24,8 +24,6 @@
import pyodbc import pyodbc
import time import time
import requests
from retrying import retry
from odbcReferences import driver from odbcReferences import driver
from config import logRetentionDays, maximumSQLAttempts, loggingMode from config import logRetentionDays, maximumSQLAttempts, loggingMode
from datetime import datetime, timedelta from datetime import datetime, timedelta
@ -101,155 +99,6 @@ class logsManager:
time.sleep(1) time.sleep(1)
@retry(stop_max_attempt_number=3, wait_fixed=1000)
def geoIPLookup(self, ip, token):
global delayUntil
try:
if 'delayUntil' in locals() and datetime.now() < delayUntil:
print("Rate limit exceeded. Please wait before trying again.")
return None
url = f"https://ipinfo.io/{ip}?token={token}"
response = requests.get(url)
response.raise_for_status()
data = response.json()
if response.status_code == 429:
print("Rate limit exceeded. Please wait before trying again.")
today = datetime.now()
if today.month == 12:
delayUntil = datetime(today.year + 1, 1, 1)
else:
delayUntil = datetime(today.year, today.month + 1, 1)
return None
geoinfo = {
"ip": data.get("ip"),
"hostname": data.get("hostname"),
"city": data.get("city"),
"region": data.get("region"),
"country": data.get("country"),
"loc": data.get("loc"),
"org": data.get("org"),
"postal": data.get("postal"),
"timezone": data.get("timezone"),
}
return geoinfo
except Exception as e:
print("An error occurred whilst doing a GeoIP lookup.", e)
raise
def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token):
currentAttempts = 1
self.insertHost(hostname, ipAddress)
self.deleteOldLogs("monutil_hostlogs", "logTime")
while currentAttempts <= maximumSQLAttempts:
try:
conn = pyodbc.connect(self.conn_str)
cursor = conn.cursor()
if int(live) == 0:
if loggingMode == 'mssql' or loggingMode == 'mariadb':
cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail)
elif loggingMode == 'postgresql':
cursor.execute("UPDATE monutil_ipblock SET live = FALSE WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = TRUE", hostname, blockedIPAddress, jail)
else:
raise Exception("A serious error has occurred. Unrecognised DBMS.")
autoIncrementID = 0
else:
if loggingMode == 'mariadb':
cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
cursor.execute("SELECT LAST_INSERT_ID()")
autoIncrementID = cursor.fetchone()[0]
elif loggingMode == 'mssql':
query = cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) OUTPUT INSERTED.logID VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
autoIncrementID = query.fetchone()[0]
elif loggingMode == 'postgresql':
cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING logID;", hostname, blockedIPAddress, jail, live, logTime)
autoIncrementID = cursor.fetchone()[0]
else:
raise Exception("A serious error has occurred. Unrecognised DBMS.")
conn.commit()
conn.close()
if autoIncrementID > 0:
geoinfo = self.geoIPLookup(blockedIPAddress, token)
self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"], geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"])
break
except pyodbc.Error as ex:
currentAttempts += 1
print("Error inserting/updating data in monutil_ipblock:", ex)
if not currentAttempts <= maximumSQLAttempts:
raise
time.sleep(1)
def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone):
currentAttempts = 1
while currentAttempts <= maximumSQLAttempts:
try:
conn = pyodbc.connect(self.conn_str)
cursor = conn.cursor()
cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID,
hostname, city, region, country, loc, org, postal, timezone)
conn.commit()
conn.close()
break
except pyodbc.Error as ex:
currentAttempts += 1
print("Error inserting data in monutil_geoip:", ex)
if not currentAttempts <= maximumSQLAttempts:
raise
time.sleep(1)
def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime): def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime):
currentAttempts = 1 currentAttempts = 1
@ -294,7 +143,7 @@ class logsManager:
oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC" oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC"
elif loggingMode == 'mariadb' or loggingMode == 'postgresql': elif loggingMode == 'mariadb':
oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1" oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1"

View file

@ -26,7 +26,6 @@ import config
odbcMariaDB = '{MariaDB}' odbcMariaDB = '{MariaDB}'
odbcMSSQL = '{ODBC Driver 17 for SQL Server}' odbcMSSQL = '{ODBC Driver 17 for SQL Server}'
odbcPostgreSQL = '{PostgreSQL}'
# DO NOT MODIFY ANYTHING BELOW THIS LINE # DO NOT MODIFY ANYTHING BELOW THIS LINE
@ -39,7 +38,3 @@ if config.loggingMode == 'mariadb':
elif config.loggingMode == 'mssql': elif config.loggingMode == 'mssql':
driver = odbcMSSQL driver = odbcMSSQL
elif config.loggingMode == 'postgresql':
driver = odbcPostgreSQL

View file

@ -75,26 +75,19 @@ class rabbitMQClient:
result = body.split('|') result = body.split('|')
if result[3] == 'cpumem': if result[3] == 'cpumem':
result[2] = datetime.datetime.fromisoformat(result[2]) result[2] = datetime.datetime.fromisoformat(result[2])
manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
manager.insertHostLog(result[0], result[1], result[2], result[4], result[5]) manager.insertHostLog(result[0], result[1], result[2], result[4], result[5])
ch.basic_ack(delivery_tag=method.delivery_tag) ch.basic_ack(delivery_tag=method.delivery_tag)
if result[3] == 'url': if result[3] == 'url':
result[2] = datetime.datetime.fromisoformat(result[2]) result[2] = datetime.datetime.fromisoformat(result[2])
manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword) manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
manager.insertURLLog(result[0], result[1], result[2], result[4], result[5]) manager.insertURLLog(result[0], result[1], result[2], result[4], result[5])
ch.basic_ack(delivery_tag=method.delivery_tag) ch.basic_ack(delivery_tag=method.delivery_tag)
if result[3] == 'ipBlock':
result[2] = datetime.datetime.fromisoformat(result[2])
manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken)
ch.basic_ack(delivery_tag=method.delivery_tag)
context = ssl.create_default_context( context = ssl.create_default_context(
cafile=self.ca) cafile=self.ca)
context.verify_mode = ssl.CERT_REQUIRED context.verify_mode = ssl.CERT_REQUIRED

View file

@ -1,116 +0,0 @@
def main():
import socket
import os
import config
import datetime
import signal
import threading
from log import logsManager
import sys
import select
socketFile = '/etc/monutil/ip.socket'
if len(sys.argv) == 4:
hostname = socket.gethostname()
ipAddress = socket.gethostbyname(hostname)
blockedIPAddress = sys.argv[1]
jail = sys.argv[2]
live = int(sys.argv[3])
logTime = datetime.datetime.now()
clientSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
clientSocket.connect(socketFile)
data = f"{hostname}|{ipAddress}|{logTime}|ipBlock|{blockedIPAddress}|{jail}|{live}"
clientSocket.send(data.encode('utf-8'))
clientSocket.close()
sys.exit(0)
else:
if os.path.exists(socketFile):
os.remove(socketFile)
stopEvent = threading.Event()
failureEvent = threading.Event()
dataBuffer = []
def publishData(stopEvent):
try:
if config.loggingMode == 'rabbitmq':
import rabbitmq
rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey)
while not (stopEvent.is_set() and not failureEvent.is_set()):
if dataBuffer:
data = dataBuffer.pop(0)
if config.loggingMode == 'rabbitmq':
rabbitmq.publish(f"{data}")
else:
print("Not yet implemented")
except Exception:
failureEvent.set()
def cleanup(signum, frame):
print("Signal received, shutting down...")
stopEvent.set()
signal.signal(signal.SIGTERM, cleanup)
signal.signal(signal.SIGINT, cleanup)
def server(stopEvent):
try:
serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
serverSocket.bind(socketFile)
serverSocket.listen(50)
while not (stopEvent.is_set() and not failureEvent.is_set()):
readable, _, _ = select.select([serverSocket], [], [], 1.0)
if readable:
clientSocket, _ = serverSocket.accept()
data = clientSocket.recv(1024).decode('utf-8')
if data:
dataBuffer.append(data)
print(data)
clientSocket.close()
except Exception:
failureEvent.set()
publishThread = threading.Thread(target=publishData, args=(stopEvent,))
serverThread = threading.Thread(target=server, args=(stopEvent,))
publishThread.start()
serverThread.start()
publishThread.join()
serverThread.join()
if failureEvent.is_set():
print("One of the threads failed. Terminating")
sys.exit(1)
if __name__ == "__main__":
main()

View file

@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
); );
CREATE TABLE monutil_hostlogs ( CREATE TABLE monutil_hostlogs (
logID BIGINT AUTO_INCREMENT PRIMARY KEY, logID INT AUTO_INCREMENT PRIMARY KEY,
hostname VARCHAR(255) NOT NULL, hostname VARCHAR(255) NOT NULL,
logTime DATETIME NOT NULL, logTime DATETIME NOT NULL,
cpu DECIMAL(5, 2) NOT NULL, cpu DECIMAL(5, 2) NOT NULL,
@ -13,34 +13,10 @@ CREATE TABLE monutil_hostlogs (
); );
CREATE TABLE monutil_urlLogs ( CREATE TABLE monutil_urlLogs (
logID BIGINT AUTO_INCREMENT PRIMARY KEY, logID INT AUTO_INCREMENT PRIMARY KEY,
hostname VARCHAR(255) NOT NULL, hostname VARCHAR(255) NOT NULL,
url TEXT NOT NULL, url TEXT NOT NULL,
logTime DATETIME NOT NULL, logTime DATETIME NOT NULL,
responseTime DECIMAL(5, 2) NOT NULL, responseTime DECIMAL(5, 2) NOT NULL,
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
); );
CREATE TABLE monutil_ipblock (
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
hostname VARCHAR(255) NOT NULL,
blockedIPAddress VARCHAR(45) NOT NULL,
jail VARCHAR(255),
live BOOLEAN NOT NULL,
logTime DATETIME NOT NULL,
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
);
CREATE TABLE monutil_geoip (
geoID BIGINT AUTO_INCREMENT PRIMARY KEY,
logID BIGINT NOT NULL,
hostname VARCHAR(255),
city VARCHAR(255),
region VARCHAR(255),
country VARCHAR(255),
loc VARCHAR(50),
org VARCHAR(255),
postal VARCHAR(20),
timezone VARCHAR(50),
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
);

View file

@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
); );
CREATE TABLE monutil_hostlogs ( CREATE TABLE monutil_hostlogs (
logID BIGINT IDENTITY(1, 1) PRIMARY KEY, logID INT IDENTITY(1, 1) PRIMARY KEY,
hostname NVARCHAR(255) NOT NULL, hostname NVARCHAR(255) NOT NULL,
logTime DATETIME NOT NULL, logTime DATETIME NOT NULL,
cpu DECIMAL(5, 2) NOT NULL, cpu DECIMAL(5, 2) NOT NULL,
@ -13,34 +13,10 @@ CREATE TABLE monutil_hostlogs (
); );
CREATE TABLE monutil_urlLogs ( CREATE TABLE monutil_urlLogs (
logID BIGINT IDENTITY(1, 1) PRIMARY KEY, logID INT IDENTITY(1, 1) PRIMARY KEY,
hostname NVARCHAR(255) NOT NULL, hostname NVARCHAR(255) NOT NULL,
url NVARCHAR(MAX) NOT NULL, url NVARCHAR(MAX) NOT NULL,
logTime DATETIME NOT NULL, logTime DATETIME NOT NULL,
responseTime DECIMAL(5, 2) NOT NULL, responseTime DECIMAL(5, 2) NOT NULL,
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname) FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
); );
CREATE TABLE monutil_ipblock (
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
hostname NVARCHAR(255) NOT NULL,
blockedIPAddress NVARCHAR(45) NOT NULL,
jail NVARCHAR(255),
live BIT NOT NULL,
logTime DATETIME NOT NULL,
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
);
CREATE TABLE monutil_geoip (
geoID BIGINT IDENTITY(1,1) PRIMARY KEY,
logID BIGINT NOT NULL,
hostname NVARCHAR(255),
city NVARCHAR(255),
region NVARCHAR(255),
country NVARCHAR(255),
loc NVARCHAR(50),
org NVARCHAR(255),
postal NVARCHAR(20),
timezone NVARCHAR(50),
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
);

View file

@ -1,46 +0,0 @@
CREATE TABLE monutil_hosts (
hostname VARCHAR(255) PRIMARY KEY,
ipAddress VARCHAR(15) NOT NULL
);
CREATE TABLE monutil_hostlogs (
logID BIGSERIAL PRIMARY KEY,
hostname VARCHAR(255) NOT NULL,
logTime TIMESTAMP NOT NULL,
cpu DECIMAL(5, 2) NOT NULL,
memory DECIMAL(5, 2) NOT NULL,
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
);
CREATE TABLE monutil_urlLogs (
logID BIGSERIAL PRIMARY KEY,
hostname VARCHAR(255) NOT NULL,
url TEXT NOT NULL,
logTime TIMESTAMP NOT NULL,
responseTime DECIMAL(5, 2) NOT NULL,
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
);
CREATE TABLE monutil_ipblock (
logID BIGSERIAL PRIMARY KEY,
hostname VARCHAR(255) NOT NULL,
blockedIPAddress VARCHAR(45) NOT NULL,
jail VARCHAR(255),
live BOOLEAN NOT NULL,
logTime TIMESTAMP NOT NULL,
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
);
CREATE TABLE monutil_geoip (
geoID BIGSERIAL PRIMARY KEY,
logID BIGINT NOT NULL,
hostname VARCHAR(255),
city VARCHAR(255),
region VARCHAR(255),
country VARCHAR(255),
loc VARCHAR(50),
org VARCHAR(255),
postal VARCHAR(20),
timezone VARCHAR(50),
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
);