Commit f0d93f23 authored by Jason Frisvold's avatar Jason Frisvold
Browse files

- Rename server_id to timer_id in timer module

- Handle disabled timers
- - Don't build configs for disabled timers
- - Send "delete" config for disabled timers
- Beginning of data processing
- - Decrypt local file
- - Parse XML to obtain results
- - Move file into central repository
- Use built-in functions for path joining
- Use config global instead of passing as variable
- Clean up formatters
parent a94b4262
......@@ -25,7 +25,7 @@ class SkynetTimer(object):
def __init__(self):
self._client_id = None
self._action = None
self._server_id = None
self._timer_id = None
self._minute = list()
self._hour = list()
self._day = list()
......@@ -37,7 +37,7 @@ class SkynetTimer(object):
def loadfromfile(self, file):
self.client_id = None
self.action = file[0].strip()
self.server_id = file[1].strip()
self.timer_id = file[1].strip()
self.minute = file[2].strip()
self.hour = file[3].strip()
self.day = file[4].strip()
......@@ -50,9 +50,9 @@ class SkynetTimer(object):
timingcursor = timingdb.cursor()
if self.action == 'add':
timingcursor.execute('INSERT INTO timer_details (server_id, '
timingcursor.execute('INSERT INTO timer_details (timer_id, '
'override_flag, ip_range, nmap_options) '
'VALUES (?, ?, ?, ?)', (self.server_id,
'VALUES (?, ?, ?, ?)', (self.timer_id,
self.override,
self.ip_range,
self.options))
......@@ -62,22 +62,22 @@ class SkynetTimer(object):
for day in self._day:
for month in self._month:
timingcursor.execute('INSERT INTO timers ('
'server_id, minute, hour, '
'timer_id, minute, hour, '
'day, month) VALUES (?, ?, ?, '
'?, ?)', (self.server_id,
'?, ?)', (self.timer_id,
minute, hour, day,
month))
elif self.action == 'modify':
timingcursor.execute('DELETE FROM timers WHERE server_id = ?',
(self.server_id,))
timingcursor.execute('DELETE FROM timers WHERE timer_id = ?',
(self.timer_id,))
timingcursor.execute('DELETE FROM timer_details WHERE server_id = '
'?', (self.server_id,))
timingcursor.execute('DELETE FROM timer_details WHERE timer_id = '
'?', (self.timer_id,))
timingcursor.execute('INSERT INTO timer_details (server_id, '
timingcursor.execute('INSERT INTO timer_details (timer_id, '
'override_flag, ip_range, nmap_options) '
'VALUES (?, ?, ?, ?)', (self.server_id,
'VALUES (?, ?, ?, ?)', (self.timer_id,
self.override,
self.ip_range,
self.options))
......@@ -87,18 +87,18 @@ class SkynetTimer(object):
for day in self._day:
for month in self._month:
timingcursor.execute('INSERT INTO timers ('
'server_id, minute, hour, '
'timer_id, minute, hour, '
'day, month) VALUES (?, ?, ?, '
'?, ?)', (self.server_id,
'?, ?)', (self.timer_id,
minute, hour, day,
month))
elif self.action == 'delete':
timingcursor.execute('DELETE FROM timers WHERE server_id = ?',
(self.server_id,))
timingcursor.execute('DELETE FROM timers WHERE timer_id = ?',
(self.timer_id,))
timingcursor.execute('DELETE FROM timer_details WHERE server_id = '
'?', (self.server_id,))
timingcursor.execute('DELETE FROM timer_details WHERE timer_id = '
'?', (self.timer_id,))
timingdb.commit()
......@@ -115,16 +115,16 @@ class SkynetTimer(object):
del self._client_id
@property
def server_id(self):
return self._server_id
def timer_id(self):
return self._timer_id
@server_id.setter
def server_id(self, value):
self._server_id = value
@timer_id.setter
def timer_id(self, value):
self._timer_id = value
@server_id.deleter
def server_id(self):
del self._server_id
@timer_id.deleter
def timer_id(self):
del self._timer_id
@property
def minute(self):
......
......@@ -36,6 +36,9 @@ dbhost = localhost
# Port where the database can be accessed
dbport = 3306
# Data directory where files are stored permanently
data_dir = /opt/skynet/data
# Working directory where files are temporarily stored until they can be
# processed
work_dir = /opt/skynet/server/working
......
......@@ -30,6 +30,10 @@ import MySQLdb
from netaddr import IPAddress
import paramiko
import tempfile
import xml.etree.ElementTree as ET
import subprocess
from subprocess import Popen
import shutil
# Global Variables
verbose = False
......@@ -127,12 +131,13 @@ def main(argv):
# Get results from server
get_results(dbhandle, server_conn, server['output_dir'],
cfg['work_dir'], str(server['id']))
str(server['id']))
server_conn.close()
# Process results, update database, store results
process_results(dbhandle)
# Sleep until the beginning of the next minute
sleeptime = 60 - datetime.utcnow().second
logger.info('Sleeping until next loop iteration - {0} seconds'
......@@ -159,7 +164,9 @@ def loadconfig():
cfg['dbport'] = config.get('skynet', 'dbport')
cfg['dbuser'] = config.get('skynet', 'dbuser')
cfg['dbpass'] = config.get('skynet', 'dbpass')
cfg['data_dir'] = config.get('skynet', 'data_dir')
cfg['work_dir'] = config.get('skynet', 'work_dir')
cfg['gpg_binary'] = config.get('skynet', 'gpg_binary')
for k, v in cfg.items():
logger.debug('{0} = {1}'.format(k, v))
......@@ -168,13 +175,14 @@ def get_servers(dbhandle):
logger.info('Loading server list')
sql = '''SELECT DISTINCT c.id, c.server_ip, c.key_type, c.ssh_key,
c.gpg_key, c.config_dir, c.output_dir, c.ssh_username
FROM cloud AS c, timers AS ti, target AS t, spawn AS s WHERE
(ti.cloud_id = c.id AND ti.spawn_id = s.id AND ti.target_id = t.id)
AND (c.last_contacted <= c.last_modified OR c.last_contacted <=
c.gpg_key, c.config_dir, c.output_dir, c.ssh_username FROM cloud AS
c, timers AS ti, target AS t, spawn AS s WHERE (ti.cloud_id = c.id
AND ti.spawn_id = s.id AND ti.target_id = t.id) AND
(c.last_contacted <= c.last_modified OR c.last_contacted <=
ti.last_modified OR c.last_contacted <= t.last_modified OR
c.last_contacted <= s.last_modified OR c.last_contacted <=
DATE_SUB(NOW(), INTERVAL c.contact_frequency MINUTE))'''
c.last_contacted <= s.last_modified OR (c.last_contacted <=
DATE_SUB(NOW(), INTERVAL c.contact_frequency MINUTE) AND
c.disabled = 0))'''
try:
cur = dbhandle.cursor()
......@@ -187,7 +195,7 @@ def get_servers(dbhandle):
rows = cur.fetchall()
if (len(rows) > 0):
for row in rows:
logger.debug('Database Retrieval : ' + str(row))
logger.debug('Database Retrieval : {0}'.format(str(row)))
serverlist.append({ 'id' : row[0], 'ip' : str(IPAddress(row[1])),
'key_type' : row[2], 'ssh_key' : row[3],
'gpg_key' : row[4], 'config_dir' : row[5],
......@@ -220,7 +228,8 @@ def build_configs(dbhandle, server_id):
logger.info('Building server configuration')
sql = '''SELECT ti.id, c.server_ip, s.options, s.override, t.address,
ti.hour, ti.minute, ti.day, ti.month FROM timers AS ti, cloud AS c,
ti.hour, ti.minute, ti.day, ti.month, c.disabled,
s.disabled, t.disabled, ti.disabled FROM timers AS ti, cloud AS c,
spawn AS s, target AS t WHERE c.id = ti.cloud_id AND s.id =
ti.spawn_id AND t.id = ti.target_id AND ti.cloud_id = %s AND
(c.last_contacted <= c.last_modified OR c.last_contacted <=
......@@ -239,11 +248,12 @@ def build_configs(dbhandle, server_id):
if (len(rows) > 0):
for row in rows:
logger.debug('Database Retrieval : {0}'.format(str(row)))
disabled = row[9] + row[10] + row[11] + row[12]
configlist.append({ 'id' : row[0], 'ip' : str(IPAddress(row[1])),
'options' : row[2], 'override' : row[3],
'address' : row[4], 'hour' : row[5],
'minute' : row[6], 'day' : row[7],
'month' : row[8]})
'month' : row[8], 'disabled' : disabled})
return configlist
......@@ -252,7 +262,10 @@ def push_configs(server_conn, configlist, config_path):
for config in configlist:
tmpfile = tempfile.NamedTemporaryFile(delete=False)
tmpfile.write("modify\n")
if (config['disabled'] > 0):
tmpfile.write("delete\n")
else:
tmpfile.write("modify\n")
tmpfile.write(str(config['id']) + "\n")
tmpfile.write(config['minute'] + "\n")
tmpfile.write(config['hour'] + "\n")
......@@ -264,14 +277,14 @@ def push_configs(server_conn, configlist, config_path):
tmpfile.close()
logger.debug('Uploading {0} to {1}'
.format(tmpfile.name, config_path + '/' +
os.path.basename(tmpfile.name) + '.skynet'))
.format(tmpfile.name, os.path.join(config_path,
os.path.basename(tmpfile.name) + '.skynet')))
server_conn.put(tmpfile.name,
config_path + '/' +
os.path.basename(tmpfile.name) + '.skynet')
os.path.join(config_path,
os.path.basename(tmpfile.name) + '.skynet'))
os.remove(tmpfile.name)
def get_results(dbhandle, server_conn, output_dir, work_dir, server_id):
def get_results(dbhandle, server_conn, output_dir, server_id):
logger.info('Retrieving results from server')
file_list = server_conn.listdir(output_dir)
......@@ -279,9 +292,10 @@ def get_results(dbhandle, server_conn, output_dir, work_dir, server_id):
logger.info('Retrieved list of files - {0}'.format(file_list))
for result in file_list:
server_conn.get(output_dir + '/' + result, work_dir)
server_conn.get(os.path.join(output_dir, result),
os.path.join(cfg['work_dir'], result))
# TODO : Validate received file prior to deleting remote file
server_conn.remove(output_dir + '/' + result)
server_conn.remove(os.path.join(output_dir, result))
logger.info('Updating last_contacted for server {0}'.format(server_id))
......@@ -294,6 +308,72 @@ def get_results(dbhandle, server_conn, output_dir, work_dir, server_id):
except MySQLdb.Error, e:
logger.exception("MySQL Error [{0}]: {1}".format(e.args[0], e.args[1]))
def process_results(dbhandle):
logger.info('Processing results')
if (os.path.exists(cfg['work_dir'])):
filelist = []
for name in os.listdir(cfg['work_dir']):
filelist.append(name)
else:
logger.critical('Working directory does not exist')
return
for nmapfile in filelist:
logger.debug('Processing file {0}'.format(os.path.join(cfg['work_dir'],
nmapfile)))
# Decrypt the file into a stream and load XML
cmd = (cfg['gpg_binary'], '--batch', '--yes', '--decrypt',
os.path.join(cfg['work_dir'], nmapfile))
decrypter = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output = decrypter.stdout.read()
nmapxml = ET.fromstring(output)
# Pull stats from results
# NMAP version (scaninfo.version)
nmapversion = nmapxml.get('version')
# Start/End Times (nmaprun.start, runstats.finished.time)
starttime = nmapxml.get('start')
finished = nmapxml.find('runstats/finished')
endtime = finished.get('time')
# Time to complete scan (runstats.finished.elapsed)
runtime = finished.get('elapsed')
# Status (runstats.finished.exit)
status = finished.get('exit')
# Number of open ports (count ports)
portcount = 0
for port in nmapxml.iter('port'):
portcount += 1
logger.debug('''Retrieved stats : version - {0}, start time - {1},
end time = {2}, elapsed time - {3}, exit status - {4},
port count - {5}'''.format(version, starttime, endtime,
runtime, status, portcount))
# Update the database with the results
# diff between previous and current stats? (ndiff?)
# Move the processed file into the central repo
scantime = datetime.fromtimestamp(int(starttime))
filedir = os.path.join(str(cfg['data_dir']), str(scantime.year),
'{:02d}'.format(scantime.month),
'{:02d}'.format(scantime.day))
if (not os.path.exists(filedir)):
os.makedirs(filedir)
logger.debug('Moving processed file to {0}'.format(filedir))
shutil.move(os.path.join(cfg['work_dir'], nmapfile), filedir)
return
###
# Usage
###
......@@ -328,4 +408,4 @@ try:
if __name__ == "__main__":
main(sys.argv[1:])
except KeyboardInterrupt:
sys.exit()
sys.exit()
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment