spawner.py 14.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#!/usr/bin/python

#* Skynet - Automated "Cloud" Security Scanner                                *#
#* Copyright (C) 2014  Jason Frisvold <friz@godshell.com>                     *#
#*                                                                            *#
#* This program is free software; you can redistribute it and/or modify       *#
#* it under the terms of the GNU General Public License as published by       *#
#* the Free Software Foundation; either version 2 of the License, or          *#
#* (at your option) any later version.                                        *#
#*                                                                            *#
#* This program is distributed in the hope that it will be useful,            *#
#* but WITHOUT ANY WARRANTY; without even the implied warranty of             *#
#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *#
#* GNU General Public License for more details.                               *#
#*                                                                            *#
#* You should have received a copy of the GNU General Public License          *#
#* along with this program; if not, write to the Free Software                *#
#* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA *#

# Import libraries
import sys
import getopt
import ConfigParser
Jason Frisvold's avatar
Jason Frisvold committed
24
25
import time
import sqlite3
26
27
import logging
from datetime import datetime
28
import os, os.path
29
from subprocess import Popen
30
import re
31
import shlex
32

33
34
sys.path.append('../libs')
from skynettimer import SkynetTimer
Jason Frisvold's avatar
Jason Frisvold committed
35
from funcs import pid_exists
36

37
38
39
40
41
# Global Variables
verbose = False
programname = 'SkyNet Spawner'
version = '1.0'
configfile = 'spawner.conf'
Jason Frisvold's avatar
Jason Frisvold committed
42
43
44
45
46
47
defaultconfigfile = 'spawner.default.conf'
cfg = {}

# Set up the logger
logging.basicConfig()
logger = logging.getLogger('spawner')
48
49
50
51
52
53

###
# Main
###
def main(argv):
    try:
54
55
56
57
        opts, args = getopt.gnu_getopt(argv,
                                       "dhvVs:c:",
                                       ["debug", "help", "verbose", "config",
                                        "version", "license"])
58
59
60
61
62
63
64
    except getopt.GetoptError, err:
        print str(err)
        usage()
        sys.exit(2)

    for o, a in opts:
        if o in ("-v", "--verbose"):
65
66
            logger.setLevel(logging.INFO)
        elif o in ("-d", "--debug"):
Jason Frisvold's avatar
Jason Frisvold committed
67
68
            logger.setLevel(logging.DEBUG)
        elif o in ("-c", "--config"):
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
            global configfile
            # TODO : This should be validated
            configfile = o
        elif o in ("-V", "--version"):
            print '{0} {1}'.format(programname, version)
            sys.exit()
        elif o in ("-h", "--help"):
            usage()
            sys.exit()
        elif o in ("--license"):
            license()
            sys.exit()
        else:
            assert False, "unhandled option"

Jason Frisvold's avatar
Jason Frisvold committed
84
    loadconfig()
85

Jason Frisvold's avatar
Jason Frisvold committed
86
87
    # Main loop
    while True:
88
89
        loopstarttime = datetime.now()

90
91
        logger.info('Loop starts at ' + str(loopstarttime))

92
93
94
95
96
        timingdb = open_timing_database()

        filelist = check_configdir()
        if (len(filelist) > 0):
            process_config(timingdb, filelist)
97

98
        spawnlist = check_timing(loopstarttime, timingdb)
Jason Frisvold's avatar
Jason Frisvold committed
99
100
101

        spawnlist = check_complete(spawnlist, timingdb)

102
        if (len(spawnlist) > 0):
103
            spawn_process(spawnlist)
104

105
106
        logger.info('Sleeping until next loop iteration')

Jason Frisvold's avatar
Jason Frisvold committed
107
108
109
        # Sleep until the beginning of the next minute
        sleeptime = 60 - datetime.utcnow().second
        time.sleep(sleeptime)
110

Jason Frisvold's avatar
Jason Frisvold committed
111
def loadconfig():
112
113
    logger.info('Loading spawner configuration')

Jason Frisvold's avatar
Jason Frisvold committed
114
115
116
117
118
119
120
    config = ConfigParser.SafeConfigParser()

    config.read(defaultconfigfile)

    try:
        config.read(configfile)
    except:
121
122
        e = sys.exc_info()[0]
        logger.exception('Unable to load config file = ' + e)
Jason Frisvold's avatar
Jason Frisvold committed
123
124
125
126
127
128

    global cfg

    cfg['configdir']    = config.get('spawner', 'configdir')
    cfg['datadir']      = config.get('spawner', 'datadir')
    cfg['timingdb']     = config.get('spawner', 'timingdb')
129
130
    cfg['nmap_binary']  = config.get('spawner', 'nmap_binary')
    cfg['output_dir']   = config.get('spawner', 'output_dir')
Jason Frisvold's avatar
Jason Frisvold committed
131
132
133
134

    logger.debug('configdir = {0}'.format(cfg['configdir']))
    logger.debug('datadir = {0}'.format(cfg['datadir']))
    logger.debug('timingdb = {0}'.format(cfg['timingdb']))
135
136
    logger.debug('nmap_binary = {0}'.format(cfg['nmap_binary']))
    logger.debug('output_dir = {0}'.format(cfg['output_dir']))
137

Jason Frisvold's avatar
Jason Frisvold committed
138
def check_configdir():
139
    logger.info('Check Config')
Jason Frisvold's avatar
Jason Frisvold committed
140
    
141
142
143
144
    if (os.path.exists(cfg['configdir'])):
        # Look for new files

        filelist = []
Jason Frisvold's avatar
Jason Frisvold committed
145

146
147
148
149
150
151
152
153
154
155
        for name in os.listdir(cfg['configdir']):
            if re.match('.*\.skynet$', name):
                filelist.append(name)

        return filelist
    else:
        logger.critical('Configuration directory does not exist')
        sys.exit()

def process_config(timingdb, filelist):
156
    logger.info('Process Config')
157

158
    for file in filelist:
159
160
161
162
163
164
        logger.debug('Processing file - ' + file)
        incoming = open(os.path.join(cfg['configdir'], file), 'r')
        lines = incoming.readlines()
        incoming.close()

        try:
165
166
167
168
169
            timer = SkynetTimer()
            timer.loadfromfile(lines)
            timer.sendtodb(timingdb)
            os.unlink(os.path.join(cfg['configdir'], file))

170
171
        except:
            e = sys.exc_info()[0]
172
            logger.exception("Error creating timing object : %s", e)
173

174
def check_timing(loopstarttime, timingdb):
175
176
    logger.info('Check Timing')

177
178
179
180
181
    spawnlist = []

    try:
        timingcursor = timingdb.cursor()

182
183
184
185
186
187
188
        timingcursor.execute('''SELECT t.server_id, td.override_flag FROM timers
                             AS t, timer_details AS td WHERE (t.minute = ? OR
                             t.minute = "*") AND (t.hour = ? OR t.hour = "*")
                             AND (t.day = ? OR t.day = "*") AND (t.month = ? OR
                             t.month = "*") AND t.server_id = td.server_id''',
                             (loopstarttime.minute, loopstarttime.hour,
                              loopstarttime.day, loopstarttime.month))
189
190
191
192
193

        rows = timingcursor.fetchall()

        if (len(rows) > 0):
            for row in rows:
Jason Frisvold's avatar
Jason Frisvold committed
194
                spawnlist.append((row[0], row[1]))
195
196
197
198

    except:
        e = sys.exc_info()[0]
        logger.exception("Error checking timing : %s", e)
199
200

    return spawnlist
Jason Frisvold's avatar
Jason Frisvold committed
201

Jason Frisvold's avatar
Jason Frisvold committed
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
def check_complete(spawnlist, timingdb):
    logger.info('Check for completed processes')
    
    try:
        timingcursor = timingdb.cursor()

        timingcursor.execute('SELECT server_id, start_time, pid FROM spawned')

        rows = timingcursor.fetchall()

    except:
        e = sys.exc_info()[0]
        logger.exception("Error getting spawned process data : %s", e)

    if (len(rows) > 0):
        for row in rows:
            logger.debug('Checking id ' + str(row[0]) + ' started at ' +
                         str(row[1]))

            if pid_exists(row[2]):
                logger.debug('PID ' + str(row[2]) + ' Still running')

                try:
                    # If server_id should be spawned, isn't allowed to override,
                    # and is still running, then remove it from the spawnlist
                    # and set overrun = 1
                    spawnlist.remove((row[0],0))
229
230
                    logger.debug('ID ' + str(row[0]) + ' not allowed to '
                                 'override, removed from spawn list')
Jason Frisvold's avatar
Jason Frisvold committed
231

232
233
                    timingcursor.execute('''UPDATE spawned SET overtime = 1
                                         WHERE server_id = ? AND pid = ?''',
Jason Frisvold's avatar
Jason Frisvold committed
234
235
236
237
238
239
                                         (row[0], row[2],))
                except:
                    pass
            else:
                logger.debug('PID ' + str(row[2]) + ' Completed')
                end_time = int(time.mktime(time.gmtime()))
240
241
                timingcursor.execute('''DELETE FROM spawned WHERE server_id = ?
                                     AND pid = ?''', (row[0], row[2]))
Jason Frisvold's avatar
Jason Frisvold committed
242

243
244
                timingcursor.execute('''INSERT INTO spawn_log (server_id,
                                     start_time, end_time) VALUES (?, ?, ?)''',
Jason Frisvold's avatar
Jason Frisvold committed
245
246
247
248
249
250
                                     (row[0], row[1], end_time))

    timingdb.commit()
    
    return spawnlist

251
def spawn_process(spawnlist):
252
    logger.info('Spawn Process')
253

Jason Frisvold's avatar
Jason Frisvold committed
254
    for (timer, override) in spawnlist:
Jason Frisvold's avatar
Jason Frisvold committed
255
256
        start_time = int(time.mktime(time.gmtime()))
        logger.debug('Spawning process for timer - ' + str(timer))
257
258
259
260
261
262
263
264

        # First fork, if pid is zero, then it's the child
        try:
            pid = os.fork()
            if pid > 0:
                # The parent process should jump to the next loop iteration
                continue
        except OSError, e:
265
266
            sys.stderr.write("fork #1 failed: (%d) %s\n" %
                             (e.errno, e.strerror))
267
268
269
270
271
272
273
274
275
276
277
278
279
280
            sys.exit(1)

        # Set the parameters for the child
        os.chdir('.')
        os.umask(0)
        os.setsid()

        # Second fork, this time the parent exits and the child becomes
        # an independent process
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
281
282
            sys.stderr.write("fork #2 failed: (%d) %s\n" %
                             (e.errno, e.strerror))
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
            sys.exit(1)

        # Set the streams for the process

        if (logger.isEnabledFor(logging.DEBUG)):
            si = file('/tmp/spawner.debug.' + str(os.getpid()), 'a+')
            so = file('/tmp/spawner.debug.' + str(os.getpid()), 'a+')
            se = file('/tmp/spawner.debug.' + str(os.getpid()), 'a+')
        else:
            si = file(os.devnull, 'w')
            so = file(os.devnull, 'w')
            se = file(os.devnull, 'w')

        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

300
301
        # Since we're in a different PID, if we passed a timingdb handle in,
        # it wouldn't be valid.  So, get a new one.
302
303
304
305
306
307
308
309
310
        try:
            timingdb = open_timing_database()
            timingcursor = timingdb.cursor()

        except:
            e = sys.exc_info()[0]
            logger.exception("Error creating cursor : %s", e)

        logger.debug('Timer number ' + str(timer))
311
312
313

        timingcursor.execute('''SELECT ip_range, nmap_options FROM timer_details
                             WHERE server_id = ?''', (timer,))
314
315
316

        nmap_options = timingcursor.fetchone()

317
318
319
320
321
322
        nmap_filename = 'nmap_xml.' + str(timer) + '.' + str(start_time) + \
                        '.xml'

        optionlist = shlex.split(cfg['nmap_binary'] + ' -oX ' +
                                 cfg['output_dir'] + nmap_filename + ' ' +
                                 nmap_options[1] + ' ' + nmap_options[0])
323
324
325

        logger.debug('Will spawn with options ' + str(optionlist))

326
        spawned = Popen(optionlist)
Jason Frisvold's avatar
Jason Frisvold committed
327
328
        logger.debug('Spawned - ' + str(spawned.pid))

329
330
331
332
        timingcursor.execute('''INSERT INTO spawned (server_id, start_time, pid,
                             overtime) VALUES (?, ?, ?, 0)''', (timer,
                                                                start_time,
                                                                spawned.pid))
333
334
335
336
337

        timingdb.commit()

        spawned.wait()
        sys.exit(0)
338

339
def open_timing_database():
340
    logger.info('Opening timing database')
341
342
    if (os.path.exists(cfg['datadir'])):
        try:
343
344
            timingdb = sqlite3.connect(os.path.join(cfg['datadir'],
                                                    cfg['timingdb']))
345
346
347
348
349

            timingcursor = timingdb.cursor()

            # Create the tables, if they don't exist already
            timingcursor.execute('''CREATE TABLE IF NOT EXISTS timers
350
                                ( server_id                 INT,
351
352
353
                                minute                      TEXT,
                                hour                        TEXT,
                                day                         TEXT,
Jason Frisvold's avatar
Jason Frisvold committed
354
355
356
357
358
                                month                       TEXT
                                )''')

            timingcursor.execute('''CREATE TABLE IF NOT EXISTS timer_details
                                ( server_id                 INT,
359
360
361
362
363
364
                                override_flag               BOOLEAN,
                                ip_range                    TEXT,
                                nmap_options                TEXT
                                )''')

            timingcursor.execute('''CREATE TABLE IF NOT EXISTS spawned
Jason Frisvold's avatar
Jason Frisvold committed
365
366
367
368
369
370
371
372
373
374
375
                                ( server_id                 INT,
                                start_time                  INT,
                                pid                         INT,
                                overtime                    BOOLEAN
                                )''')

            timingcursor.execute('''CREATE TABLE IF NOT EXISTS spawn_log
                                (server_id                   INT,
                                start_time                  INT,
                                end_time                    INT,
                                status                      INT
376
377
378
379
                                )''')

        except:
            e = sys.exc_info()[0]
380
            logger.exception("Error accessing/creating sqlite database : %s", e)
381
382
383
384
385
386
387
            sys.exit()

        return(timingdb)
    else:
        logger.critical('Data Directory does not exist')
        sys.exit()

388
389
390
391
392
393
394
###
# Usage
###
def usage():
    print ('Usage: ' + sys.argv[0] + ' [OPTION]... ');
    print '{0}'.format(programname)
    print
Jason Frisvold's avatar
Jason Frisvold committed
395
396
    print ('Mandatory arguments to long options are mandatory for short ' \
           'options too.')
397
    print ('  -c  <file>     configuration file')
398
    print ('  -d, --debug    debug')
399
400
401
402
403
404
405
406
407
408
    print ('  -h, --help     display this help and exit')
    print ('  -v, --verbose  verbose output')
    print ('  -V, --version  output version information and exit')
    print ('      --license  output license information and exit')

###
# License
###
def license():
    print('Copyright (C) 2014  Jason Frisvold <friz@godshell.com>');
Jason Frisvold's avatar
Jason Frisvold committed
409
410
    print('License GPLv2+: GNU GPL version 2 or later ' \
          '<http://gnu.org/licenses/gpl.html>.');
411
412
413
414
415
416
417
418
419
420
421
422
    print('This is free software: you are free to change and redistribute it.');
    print('There is NO WARRANTY, to the extent permitted by law.');
    print('');
    print('Written by Jason Frisvold.');

###
# Run the main program, pass in the CLI arguments (minus the command name)
try:
    if __name__ == "__main__":
        main(sys.argv[1:])
except KeyboardInterrupt:
    sys.exit()