Compare commits
No commits in common. "ef294aa1c5d41f900790ba807cfcf545061d2d4c" and "bed6ff0b94d54f971e158c8c93803cc7cbd8fe13" have entirely different histories.
ef294aa1c5
...
bed6ff0b94
7 changed files with 7 additions and 380 deletions
153
log.py
153
log.py
|
|
@ -24,8 +24,6 @@
|
|||
|
||||
import pyodbc
|
||||
import time
|
||||
import requests
|
||||
from retrying import retry
|
||||
from odbcReferences import driver
|
||||
from config import logRetentionDays, maximumSQLAttempts, loggingMode
|
||||
from datetime import datetime, timedelta
|
||||
|
|
@ -101,155 +99,6 @@ class logsManager:
|
|||
|
||||
time.sleep(1)
|
||||
|
||||
@retry(stop_max_attempt_number=3, wait_fixed=1000)
|
||||
def geoIPLookup(self, ip, token):
|
||||
|
||||
global delayUntil
|
||||
|
||||
try:
|
||||
|
||||
if 'delayUntil' in locals() and datetime.now() < delayUntil:
|
||||
|
||||
print("Rate limit exceeded. Please wait before trying again.")
|
||||
return None
|
||||
|
||||
url = f"https://ipinfo.io/{ip}?token={token}"
|
||||
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
if response.status_code == 429:
|
||||
|
||||
print("Rate limit exceeded. Please wait before trying again.")
|
||||
today = datetime.now()
|
||||
|
||||
if today.month == 12:
|
||||
|
||||
delayUntil = datetime(today.year + 1, 1, 1)
|
||||
|
||||
else:
|
||||
|
||||
delayUntil = datetime(today.year, today.month + 1, 1)
|
||||
|
||||
return None
|
||||
|
||||
geoinfo = {
|
||||
"ip": data.get("ip"),
|
||||
"hostname": data.get("hostname"),
|
||||
"city": data.get("city"),
|
||||
"region": data.get("region"),
|
||||
"country": data.get("country"),
|
||||
"loc": data.get("loc"),
|
||||
"org": data.get("org"),
|
||||
"postal": data.get("postal"),
|
||||
"timezone": data.get("timezone"),
|
||||
}
|
||||
|
||||
|
||||
return geoinfo
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("An error occurred whilst doing a GeoIP lookup.", e)
|
||||
raise
|
||||
|
||||
def insertIPBlock(self, hostname, ipAddress, blockedIPAddress, jail, live, logTime, token):
|
||||
|
||||
currentAttempts = 1
|
||||
|
||||
self.insertHost(hostname, ipAddress)
|
||||
self.deleteOldLogs("monutil_hostlogs", "logTime")
|
||||
|
||||
while currentAttempts <= maximumSQLAttempts:
|
||||
|
||||
try:
|
||||
|
||||
conn = pyodbc.connect(self.conn_str)
|
||||
cursor = conn.cursor()
|
||||
|
||||
if int(live) == 0:
|
||||
|
||||
if loggingMode == 'mssql' or loggingMode == 'mariadb':
|
||||
cursor.execute("UPDATE monutil_ipblock SET live = 0 WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = 1", hostname, blockedIPAddress, jail)
|
||||
elif loggingMode == 'postgresql':
|
||||
cursor.execute("UPDATE monutil_ipblock SET live = FALSE WHERE hostname = ? AND blockedIPAddress = ? AND jail = ? AND live = TRUE", hostname, blockedIPAddress, jail)
|
||||
else:
|
||||
raise Exception("A serious error has occurred. Unrecognised DBMS.")
|
||||
|
||||
autoIncrementID = 0
|
||||
|
||||
else:
|
||||
|
||||
if loggingMode == 'mariadb':
|
||||
|
||||
cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
|
||||
cursor.execute("SELECT LAST_INSERT_ID()")
|
||||
autoIncrementID = cursor.fetchone()[0]
|
||||
|
||||
elif loggingMode == 'mssql':
|
||||
|
||||
query = cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) OUTPUT INSERTED.logID VALUES (?, ?, ?, ?, ?)", hostname, blockedIPAddress, jail, live, logTime)
|
||||
autoIncrementID = query.fetchone()[0]
|
||||
|
||||
elif loggingMode == 'postgresql':
|
||||
|
||||
cursor.execute("INSERT INTO monutil_ipblock (hostname, blockedIPAddress, jail, live, logTime) VALUES (?, ?, ?, ?, ?) RETURNING logID;", hostname, blockedIPAddress, jail, live, logTime)
|
||||
autoIncrementID = cursor.fetchone()[0]
|
||||
|
||||
else:
|
||||
|
||||
raise Exception("A serious error has occurred. Unrecognised DBMS.")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
if autoIncrementID > 0:
|
||||
|
||||
geoinfo = self.geoIPLookup(blockedIPAddress, token)
|
||||
self.insertGeoIPInfo(autoIncrementID, geoinfo["hostname"], geoinfo["city"], geoinfo["region"], geoinfo["country"], geoinfo["loc"], geoinfo["org"], geoinfo["postal"], geoinfo["timezone"])
|
||||
|
||||
|
||||
break
|
||||
|
||||
except pyodbc.Error as ex:
|
||||
|
||||
currentAttempts += 1
|
||||
print("Error inserting/updating data in monutil_ipblock:", ex)
|
||||
|
||||
if not currentAttempts <= maximumSQLAttempts:
|
||||
raise
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
def insertGeoIPInfo(self, logID, hostname, city, region, country, loc, org, postal, timezone):
|
||||
|
||||
currentAttempts = 1
|
||||
|
||||
while currentAttempts <= maximumSQLAttempts:
|
||||
|
||||
try:
|
||||
conn = pyodbc.connect(self.conn_str)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("INSERT INTO monutil_geoip (logID, hostname, city, region, country, loc, org, postal, timezone) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", logID,
|
||||
hostname, city, region, country, loc, org, postal, timezone)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
break
|
||||
|
||||
except pyodbc.Error as ex:
|
||||
|
||||
currentAttempts += 1
|
||||
print("Error inserting data in monutil_geoip:", ex)
|
||||
|
||||
if not currentAttempts <= maximumSQLAttempts:
|
||||
raise
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def insertURLLog(self, hostname, ipAddress, log_time, url, responseTime):
|
||||
|
||||
currentAttempts = 1
|
||||
|
|
@ -294,7 +143,7 @@ class logsManager:
|
|||
|
||||
oldestLogQuery = f"SELECT TOP 1 {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC"
|
||||
|
||||
elif loggingMode == 'mariadb' or loggingMode == 'postgresql':
|
||||
elif loggingMode == 'mariadb':
|
||||
|
||||
oldestLogQuery = f"SELECT {logTimeColumn} FROM {tableName} ORDER BY {logTimeColumn} ASC LIMIT 1"
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import config
|
|||
|
||||
odbcMariaDB = '{MariaDB}'
|
||||
odbcMSSQL = '{ODBC Driver 17 for SQL Server}'
|
||||
odbcPostgreSQL = '{PostgreSQL}'
|
||||
|
||||
# DO NOT MODIFY ANYTHING BELOW THIS LINE
|
||||
|
||||
|
|
@ -39,7 +38,3 @@ if config.loggingMode == 'mariadb':
|
|||
elif config.loggingMode == 'mssql':
|
||||
|
||||
driver = odbcMSSQL
|
||||
|
||||
elif config.loggingMode == 'postgresql':
|
||||
|
||||
driver = odbcPostgreSQL
|
||||
|
|
|
|||
|
|
@ -88,13 +88,6 @@ class rabbitMQClient:
|
|||
manager.insertURLLog(result[0], result[1], result[2], result[4], result[5])
|
||||
ch.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
if result[3] == 'ipBlock':
|
||||
|
||||
result[2] = datetime.datetime.fromisoformat(result[2])
|
||||
manager = log.logsManager(config.sqlServer, config.sqlDatabase, config.sqlUsername, config.sqlPassword)
|
||||
manager.insertIPBlock(result[0], result[1], result[4], result[5], result[6], result[2], config.ipinfoAPIToken)
|
||||
ch.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
context = ssl.create_default_context(
|
||||
cafile=self.ca)
|
||||
context.verify_mode = ssl.CERT_REQUIRED
|
||||
|
|
|
|||
116
reportIPBlock.py
116
reportIPBlock.py
|
|
@ -1,116 +0,0 @@
|
|||
def main():
|
||||
|
||||
import socket
|
||||
import os
|
||||
import config
|
||||
import datetime
|
||||
import signal
|
||||
import threading
|
||||
from log import logsManager
|
||||
import sys
|
||||
import select
|
||||
|
||||
socketFile = '/etc/monutil/ip.socket'
|
||||
|
||||
if len(sys.argv) == 4:
|
||||
|
||||
hostname = socket.gethostname()
|
||||
ipAddress = socket.gethostbyname(hostname)
|
||||
blockedIPAddress = sys.argv[1]
|
||||
jail = sys.argv[2]
|
||||
live = int(sys.argv[3])
|
||||
logTime = datetime.datetime.now()
|
||||
|
||||
clientSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
clientSocket.connect(socketFile)
|
||||
data = f"{hostname}|{ipAddress}|{logTime}|ipBlock|{blockedIPAddress}|{jail}|{live}"
|
||||
clientSocket.send(data.encode('utf-8'))
|
||||
clientSocket.close()
|
||||
sys.exit(0)
|
||||
|
||||
else:
|
||||
|
||||
if os.path.exists(socketFile):
|
||||
os.remove(socketFile)
|
||||
|
||||
stopEvent = threading.Event()
|
||||
failureEvent = threading.Event()
|
||||
|
||||
dataBuffer = []
|
||||
|
||||
def publishData(stopEvent):
|
||||
|
||||
try:
|
||||
|
||||
if config.loggingMode == 'rabbitmq':
|
||||
|
||||
import rabbitmq
|
||||
rabbitmq = rabbitmq.rabbitMQClient(config.rabbitmqca,config.rabbitmqcacert,config.rabbitmqcakey,config.rabbitmqHost,config.rabbitmqPort,config.rabbitmqRoutingKey)
|
||||
|
||||
while not (stopEvent.is_set() and not failureEvent.is_set()):
|
||||
|
||||
if dataBuffer:
|
||||
|
||||
data = dataBuffer.pop(0)
|
||||
|
||||
if config.loggingMode == 'rabbitmq':
|
||||
|
||||
rabbitmq.publish(f"{data}")
|
||||
|
||||
else:
|
||||
|
||||
print("Not yet implemented")
|
||||
|
||||
except Exception:
|
||||
failureEvent.set()
|
||||
|
||||
def cleanup(signum, frame):
|
||||
|
||||
print("Signal received, shutting down...")
|
||||
stopEvent.set()
|
||||
|
||||
signal.signal(signal.SIGTERM, cleanup)
|
||||
signal.signal(signal.SIGINT, cleanup)
|
||||
|
||||
def server(stopEvent):
|
||||
|
||||
try:
|
||||
|
||||
serverSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
serverSocket.bind(socketFile)
|
||||
serverSocket.listen(50)
|
||||
|
||||
while not (stopEvent.is_set() and not failureEvent.is_set()):
|
||||
|
||||
readable, _, _ = select.select([serverSocket], [], [], 1.0)
|
||||
|
||||
if readable:
|
||||
clientSocket, _ = serverSocket.accept()
|
||||
data = clientSocket.recv(1024).decode('utf-8')
|
||||
|
||||
if data:
|
||||
|
||||
dataBuffer.append(data)
|
||||
print(data)
|
||||
|
||||
clientSocket.close()
|
||||
|
||||
except Exception:
|
||||
|
||||
failureEvent.set()
|
||||
|
||||
publishThread = threading.Thread(target=publishData, args=(stopEvent,))
|
||||
serverThread = threading.Thread(target=server, args=(stopEvent,))
|
||||
|
||||
publishThread.start()
|
||||
serverThread.start()
|
||||
|
||||
publishThread.join()
|
||||
serverThread.join()
|
||||
|
||||
if failureEvent.is_set():
|
||||
print("One of the threads failed. Terminating")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_hostlogs (
|
||||
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
logID INT AUTO_INCREMENT PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
cpu DECIMAL(5, 2) NOT NULL,
|
||||
|
|
@ -13,34 +13,10 @@ CREATE TABLE monutil_hostlogs (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_urlLogs (
|
||||
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
logID INT AUTO_INCREMENT PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
responseTime DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_ipblock (
|
||||
logID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
blockedIPAddress VARCHAR(45) NOT NULL,
|
||||
jail VARCHAR(255),
|
||||
live BOOLEAN NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_geoip (
|
||||
geoID BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
logID BIGINT NOT NULL,
|
||||
hostname VARCHAR(255),
|
||||
city VARCHAR(255),
|
||||
region VARCHAR(255),
|
||||
country VARCHAR(255),
|
||||
loc VARCHAR(50),
|
||||
org VARCHAR(255),
|
||||
postal VARCHAR(20),
|
||||
timezone VARCHAR(50),
|
||||
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
|
||||
);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ CREATE TABLE monutil_hosts (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_hostlogs (
|
||||
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
|
||||
logID INT IDENTITY(1, 1) PRIMARY KEY,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
cpu DECIMAL(5, 2) NOT NULL,
|
||||
|
|
@ -13,34 +13,10 @@ CREATE TABLE monutil_hostlogs (
|
|||
);
|
||||
|
||||
CREATE TABLE monutil_urlLogs (
|
||||
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
|
||||
logID INT IDENTITY(1, 1) PRIMARY KEY,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
url NVARCHAR(MAX) NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
responseTime DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_ipblock (
|
||||
logID BIGINT IDENTITY(1, 1) PRIMARY KEY,
|
||||
hostname NVARCHAR(255) NOT NULL,
|
||||
blockedIPAddress NVARCHAR(45) NOT NULL,
|
||||
jail NVARCHAR(255),
|
||||
live BIT NOT NULL,
|
||||
logTime DATETIME NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_geoip (
|
||||
geoID BIGINT IDENTITY(1,1) PRIMARY KEY,
|
||||
logID BIGINT NOT NULL,
|
||||
hostname NVARCHAR(255),
|
||||
city NVARCHAR(255),
|
||||
region NVARCHAR(255),
|
||||
country NVARCHAR(255),
|
||||
loc NVARCHAR(50),
|
||||
org NVARCHAR(255),
|
||||
postal NVARCHAR(20),
|
||||
timezone NVARCHAR(50),
|
||||
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,46 +0,0 @@
|
|||
CREATE TABLE monutil_hosts (
|
||||
hostname VARCHAR(255) PRIMARY KEY,
|
||||
ipAddress VARCHAR(15) NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_hostlogs (
|
||||
logID BIGSERIAL PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
logTime TIMESTAMP NOT NULL,
|
||||
cpu DECIMAL(5, 2) NOT NULL,
|
||||
memory DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_urlLogs (
|
||||
logID BIGSERIAL PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
logTime TIMESTAMP NOT NULL,
|
||||
responseTime DECIMAL(5, 2) NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_ipblock (
|
||||
logID BIGSERIAL PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
blockedIPAddress VARCHAR(45) NOT NULL,
|
||||
jail VARCHAR(255),
|
||||
live BOOLEAN NOT NULL,
|
||||
logTime TIMESTAMP NOT NULL,
|
||||
FOREIGN KEY (hostname) REFERENCES monutil_hosts(hostname)
|
||||
);
|
||||
|
||||
CREATE TABLE monutil_geoip (
|
||||
geoID BIGSERIAL PRIMARY KEY,
|
||||
logID BIGINT NOT NULL,
|
||||
hostname VARCHAR(255),
|
||||
city VARCHAR(255),
|
||||
region VARCHAR(255),
|
||||
country VARCHAR(255),
|
||||
loc VARCHAR(50),
|
||||
org VARCHAR(255),
|
||||
postal VARCHAR(20),
|
||||
timezone VARCHAR(50),
|
||||
FOREIGN KEY (logID) REFERENCES monutil_ipblock(logID)
|
||||
);
|
||||
Loading…
Reference in a new issue