spawner.py 15.5 KB
Newer Older
1
2
3
#!/usr/bin/python

#* Skynet - Automated "Cloud" Security Scanner                                *#
Jason Frisvold's avatar
Jason Frisvold committed
4
#* Copyright (C) 2014-present  Jason Frisvold <friz@godshell.com>             *#
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#*                                                                            *#
#* This program is free software; you can redistribute it and/or modify       *#
#* it under the terms of the GNU General Public License as published by       *#
#* the Free Software Foundation; either version 2 of the License, or          *#
#* (at your option) any later version.                                        *#
#*                                                                            *#
#* This program is distributed in the hope that it will be useful,            *#
#* but WITHOUT ANY WARRANTY; without even the implied warranty of             *#
#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *#
#* GNU General Public License for more details.                               *#
#*                                                                            *#
#* You should have received a copy of the GNU General Public License          *#
#* along with this program; if not, write to the Free Software                *#
#* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA *#

# Import libraries
import sys
import getopt
import ConfigParser
Jason Frisvold's avatar
Jason Frisvold committed
24
25
import time
import sqlite3
26
27
import logging
from datetime import datetime
28
import os, os.path
29
from subprocess import Popen
30
import re
31
import shlex
32
import tempfile
33

34
35
sys.path.append('../libs')
from skynettimer import SkynetTimer
Jason Frisvold's avatar
Jason Frisvold committed
36
from funcs import pid_exists
37

38
39
40
41
42
# Global Variables
verbose = False
programname = 'SkyNet Spawner'
version = '1.0'
configfile = 'spawner.conf'
Jason Frisvold's avatar
Jason Frisvold committed
43
44
45
46
47
48
defaultconfigfile = 'spawner.default.conf'
cfg = {}

# Set up the logger
logging.basicConfig()
logger = logging.getLogger('spawner')
49
50
51
52
53
54

###
# Main
###
def main(argv):
    try:
55
56
57
58
        opts, args = getopt.gnu_getopt(argv,
                                       "dhvVs:c:",
                                       ["debug", "help", "verbose", "config",
                                        "version", "license"])
59
60
61
62
63
64
65
    except getopt.GetoptError, err:
        print str(err)
        usage()
        sys.exit(2)

    for o, a in opts:
        if o in ("-v", "--verbose"):
66
67
            logger.setLevel(logging.INFO)
        elif o in ("-d", "--debug"):
Jason Frisvold's avatar
Jason Frisvold committed
68
69
            logger.setLevel(logging.DEBUG)
        elif o in ("-c", "--config"):
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
            global configfile
            # TODO : This should be validated
            configfile = o
        elif o in ("-V", "--version"):
            print '{0} {1}'.format(programname, version)
            sys.exit()
        elif o in ("-h", "--help"):
            usage()
            sys.exit()
        elif o in ("--license"):
            license()
            sys.exit()
        else:
            assert False, "unhandled option"

Jason Frisvold's avatar
Jason Frisvold committed
85
    loadconfig()
86

Jason Frisvold's avatar
Jason Frisvold committed
87
88
    # Main loop
    while True:
89
90
        loopstarttime = datetime.now()

91
        logger.info('Loop starts at {0}'.format(str(loopstarttime)))
92

93
94
95
96
97
        timingdb = open_timing_database()

        filelist = check_configdir()
        if (len(filelist) > 0):
            process_config(timingdb, filelist)
98

99
        spawnlist = check_timing(loopstarttime, timingdb)
Jason Frisvold's avatar
Jason Frisvold committed
100
101
102

        spawnlist = check_complete(spawnlist, timingdb)

103
        if (len(spawnlist) > 0):
104
            spawn_process(spawnlist)
105

106
107
108
109
110
111
        # Check for child exit status so we don't have zombies
        try:
            os.waitpid(-1, os.WNOHANG)
        except:
            pass

112
113
        logger.info('Sleeping until next loop iteration')

Jason Frisvold's avatar
Jason Frisvold committed
114
115
116
        # Sleep until the beginning of the next minute
        sleeptime = 60 - datetime.utcnow().second
        time.sleep(sleeptime)
117

Jason Frisvold's avatar
Jason Frisvold committed
118
def loadconfig():
119
120
    logger.info('Loading spawner configuration')

Jason Frisvold's avatar
Jason Frisvold committed
121
122
123
124
125
126
127
    config = ConfigParser.SafeConfigParser()

    config.read(defaultconfigfile)

    try:
        config.read(configfile)
    except:
128
        e = sys.exc_info()[0]
129
        logger.exception('Unable to load config file : {0}'.format(e))
Jason Frisvold's avatar
Jason Frisvold committed
130
131
132
133
134
135

    global cfg

    cfg['configdir']    = config.get('spawner', 'configdir')
    cfg['datadir']      = config.get('spawner', 'datadir')
    cfg['timingdb']     = config.get('spawner', 'timingdb')
136
137
    cfg['nmap_binary']  = config.get('spawner', 'nmap_binary')
    cfg['output_dir']   = config.get('spawner', 'output_dir')
138
139
    cfg['gpg_binary']   = config.get('spawner', 'gpg_binary')
    cfg['gpg_key']      = config.get('spawner', 'gpg_key')
Jason Frisvold's avatar
Jason Frisvold committed
140
141
142
143

    logger.debug('configdir = {0}'.format(cfg['configdir']))
    logger.debug('datadir = {0}'.format(cfg['datadir']))
    logger.debug('timingdb = {0}'.format(cfg['timingdb']))
144
145
    logger.debug('nmap_binary = {0}'.format(cfg['nmap_binary']))
    logger.debug('output_dir = {0}'.format(cfg['output_dir']))
146

Jason Frisvold's avatar
Jason Frisvold committed
147
def check_configdir():
148
    logger.info('Check Config')
149

150
151
152
153
    if (os.path.exists(cfg['configdir'])):
        # Look for new files

        filelist = []
Jason Frisvold's avatar
Jason Frisvold committed
154

155
156
157
158
159
160
161
162
163
164
        for name in os.listdir(cfg['configdir']):
            if re.match('.*\.skynet$', name):
                filelist.append(name)

        return filelist
    else:
        logger.critical('Configuration directory does not exist')
        sys.exit()

def process_config(timingdb, filelist):
165
    logger.info('Process Config')
166

167
    for file in filelist:
168
        logger.debug('Processing file - {0}'.format(file))
169
170
171
172
173
        incoming = open(os.path.join(cfg['configdir'], file), 'r')
        lines = incoming.readlines()
        incoming.close()

        try:
174
175
176
177
178
            timer = SkynetTimer()
            timer.loadfromfile(lines)
            timer.sendtodb(timingdb)
            os.unlink(os.path.join(cfg['configdir'], file))

179
180
        except:
            e = sys.exc_info()[0]
181
            logger.exception('Error creating timing object : {0}'.format(e))
182

183
def check_timing(loopstarttime, timingdb):
184
185
    logger.info('Check Timing')

186
187
188
189
190
    spawnlist = []

    try:
        timingcursor = timingdb.cursor()

191
        sql = '''SELECT t.timer_id, td.override_flag FROM timers AS t,
192
193
                 timer_details AS td WHERE (t.minute = ? OR t.minute = "*") AND
                 (t.hour = ? OR t.hour = "*") AND (t.day = ? OR t.day = "*")
194
195
                 AND (t.month = ? OR t.month = "*") AND t.timer_id =
                 td.timer_id'''
196
197
198

        timingcursor.execute(sql, (loopstarttime.minute, loopstarttime.hour,
                             loopstarttime.day, loopstarttime.month))
199
200
201
202
203

        rows = timingcursor.fetchall()

        if (len(rows) > 0):
            for row in rows:
Jason Frisvold's avatar
Jason Frisvold committed
204
                spawnlist.append((row[0], row[1]))
205
206
207

    except:
        e = sys.exc_info()[0]
208
        logger.exception('Error checking timing : {0}'.format(e))
209
210

    return spawnlist
Jason Frisvold's avatar
Jason Frisvold committed
211

Jason Frisvold's avatar
Jason Frisvold committed
212
213
def check_complete(spawnlist, timingdb):
    logger.info('Check for completed processes')
214

Jason Frisvold's avatar
Jason Frisvold committed
215
216
217
    try:
        timingcursor = timingdb.cursor()

218
        sql = 'SELECT timer_id, start_time, pid FROM spawned'
219
220

        timingcursor.execute(sql)
Jason Frisvold's avatar
Jason Frisvold committed
221
222
223
224
225

        rows = timingcursor.fetchall()

    except:
        e = sys.exc_info()[0]
226
        logger.exception('Error getting spawned process data : {0}'.format(e))
Jason Frisvold's avatar
Jason Frisvold committed
227
228
229

    if (len(rows) > 0):
        for row in rows:
230
231
            logger.debug('Checking id {0} started at {1}'.format(str(row[0]),
                                                                 str(row[1])))
Jason Frisvold's avatar
Jason Frisvold committed
232
233

            if pid_exists(row[2]):
234
                logger.debug('PID {0} Still running'.format(str(row[2])))
Jason Frisvold's avatar
Jason Frisvold committed
235
236

                try:
237
                    # If timer_id should be spawned, isn't allowed to override,
Jason Frisvold's avatar
Jason Frisvold committed
238
239
240
                    # and is still running, then remove it from the spawnlist
                    # and set overrun = 1
                    spawnlist.remove((row[0],0))
241
242
                    logger.debug('''ID {0} not allowed to override, removed from
                                    spawn list'''.format(str(row[0])))
Jason Frisvold's avatar
Jason Frisvold committed
243

244
                    sql = '''UPDATE spawned SET overtime = 1 WHERE timer_id =
245
246
247
                             ? AND pid = ?'''

                    timingcursor.execute(sql, (row[0], row[2],))
Jason Frisvold's avatar
Jason Frisvold committed
248
249
250
                except:
                    pass
            else:
251
                logger.debug('PID {0} Completed'.format(str(row[2])))
Jason Frisvold's avatar
Jason Frisvold committed
252
253
                end_time = int(time.mktime(time.gmtime()))

254
                sql = '''DELETE FROM spawned WHERE timer_id = ? AND pid = ?'''
255
256
                timingcursor.execute(sql, (row[0], row[2]))

257
                sql = '''INSERT INTO spawn_log (timer_id, start_time, end_time)
258
259
                         VALUES (?, ?, ?)'''
                timingcursor.execute(sql, (row[0], row[1], end_time))
Jason Frisvold's avatar
Jason Frisvold committed
260
261

    timingdb.commit()
262

Jason Frisvold's avatar
Jason Frisvold committed
263
264
    return spawnlist

265
def spawn_process(spawnlist):
266
    logger.info('Spawn Process')
267

Jason Frisvold's avatar
Jason Frisvold committed
268
    for (timer, override) in spawnlist:
Jason Frisvold's avatar
Jason Frisvold committed
269
        start_time = int(time.mktime(time.gmtime()))
270
        logger.debug('Spawning process for timer : {0}'.format(str(timer)))
271
272
273
274
275
276
277
278

        # First fork, if pid is zero, then it's the child
        try:
            pid = os.fork()
            if pid > 0:
                # The parent process should jump to the next loop iteration
                continue
        except OSError, e:
279
280
            sys.stderr.write('fork #1 failed: ({0}) {1}'.format(e.errno,
                                                                e.strerror))
281
282
283
284
285
286
287
288
289
290
291
292
            sys.exit(1)

        # Set the parameters for the child
        os.chdir('.')
        os.umask(0)
        os.setsid()

        # Second fork, this time the parent exits and the child becomes
        # an independent process
        try:
            pid = os.fork()
            if pid > 0:
293
                sys.exit(0)
294
        except OSError, e:
295
296
            sys.stderr.write('fork #2 failed: ({0}) {1}'.format(e.errno,
                                                                e.strerror))
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
            sys.exit(1)

        # Set the streams for the process

        if (logger.isEnabledFor(logging.DEBUG)):
            si = file('/tmp/spawner.debug.' + str(os.getpid()), 'a+')
            so = file('/tmp/spawner.debug.' + str(os.getpid()), 'a+')
            se = file('/tmp/spawner.debug.' + str(os.getpid()), 'a+')
        else:
            si = file(os.devnull, 'w')
            so = file(os.devnull, 'w')
            se = file(os.devnull, 'w')

        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

314
315
        # Since we're in a different PID, if we passed a timingdb handle in,
        # it wouldn't be valid.  So, get a new one.
316
317
318
319
320
321
        try:
            timingdb = open_timing_database()
            timingcursor = timingdb.cursor()

        except:
            e = sys.exc_info()[0]
322
            logger.exception('Error creating cursor : {0}'.format(e))
323

324
        logger.debug('Timer number {0}'.format(str(timer)))
325

326
        sql = '''SELECT ip_range, nmap_options FROM timer_details WHERE
327
                 timer_id = ?'''
328
        timingcursor.execute(sql, (timer,))
329
330
331

        nmap_options = timingcursor.fetchone()

332
        fd, tmpfile = tempfile.mkstemp()
333

334
335
336
337
338
339
340
341
        cmdline = cfg['nmap_binary'] + ' -oX ' + tmpfile + ' ' + \
                  nmap_options[1] + ' ' + nmap_options[0]

        # Prior to 2.7.3, shlex wouldn't handle unicode.
        if (sys.version_info < (2, 7, 3)):
            cmdline = cmdline.encode('ascii')

        optionlist = shlex.split(cmdline)
342

343
        logger.debug('Will spawn with options {0}'.format(str(optionlist)))
344

345
        spawned = Popen(optionlist)
346
        logger.debug('Spawned - {0}'.format(str(spawned.pid)))
Jason Frisvold's avatar
Jason Frisvold committed
347

348
        sql = '''INSERT INTO spawned (timer_id, start_time, pid, overtime)
349
350
                 VALUES (?, ?, ?, 0)'''
        timingcursor.execute(sql, (timer, start_time, spawned.pid))
351
352
353
354

        timingdb.commit()

        spawned.wait()
355
        logger.debug('Timer {0} completed.'.format(str(timer)))
356

357
358
        nmap_filename = os.path.join(cfg['output_dir'], 'nmap_xml.' + str(timer)
                                     + '.' +  str(start_time) + '.xml.gpg')
359
360
361
362

        cmd = (cfg['gpg_binary'], '--batch', '--yes', '--output', nmap_filename,
               '--encrypt', '--recipient', cfg['gpg_key'], tmpfile)

363
        logger.debug('Encrypting {0} via GPG ({1})'.format(tmpfile, str(cmd)))
364
365
366

        Popen(cmd).wait()

367
        logger.debug('Deleting temporary file - {0}'.format(tmpfile))
368
369

        os.remove(tmpfile)
370

371
        sys.exit(0)
372

373
def open_timing_database():
374
    logger.info('Opening timing database')
375
376
    if (os.path.exists(cfg['datadir'])):
        try:
377
378
            timingdb = sqlite3.connect(os.path.join(cfg['datadir'],
                                                    cfg['timingdb']))
379
380
381
382

            timingcursor = timingdb.cursor()

            # Create the tables, if they don't exist already
383
            sql = '''CREATE TABLE IF NOT EXISTS timers
384
                                ( timer_id                  INT,
385
386
387
                                minute                      TEXT,
                                hour                        TEXT,
                                day                         TEXT,
Jason Frisvold's avatar
Jason Frisvold committed
388
                                month                       TEXT
389
390
                                )'''
            timingcursor.execute(sql)
Jason Frisvold's avatar
Jason Frisvold committed
391

392
            sql = '''CREATE TABLE IF NOT EXISTS timer_details
393
                                ( timer_id                  INT,
394
395
396
                                override_flag               BOOLEAN,
                                ip_range                    TEXT,
                                nmap_options                TEXT
397
398
                                )'''
            timingcursor.execute(sql)
399

400
            sql = '''CREATE TABLE IF NOT EXISTS spawned
401
                                ( timer_id                  INT,
Jason Frisvold's avatar
Jason Frisvold committed
402
403
404
                                start_time                  INT,
                                pid                         INT,
                                overtime                    BOOLEAN
405
406
                                )'''
            timingcursor.execute(sql)
Jason Frisvold's avatar
Jason Frisvold committed
407

408
            sql = '''CREATE TABLE IF NOT EXISTS spawn_log
409
                                ( timer_id                  INT,
Jason Frisvold's avatar
Jason Frisvold committed
410
411
412
                                start_time                  INT,
                                end_time                    INT,
                                status                      INT
413
414
                                )'''
            timingcursor.execute(sql)
415
416
417

        except:
            e = sys.exc_info()[0]
418
419
            logger.exception('Error accessing/creating sqlite database : {0}'
                             .format(e))
420
421
422
423
424
425
426
            sys.exit()

        return(timingdb)
    else:
        logger.critical('Data Directory does not exist')
        sys.exit()

427
428
429
430
431
432
433
###
# Usage
###
def usage():
    print ('Usage: ' + sys.argv[0] + ' [OPTION]... ');
    print '{0}'.format(programname)
    print
Jason Frisvold's avatar
Jason Frisvold committed
434
435
    print ('Mandatory arguments to long options are mandatory for short ' \
           'options too.')
436
    print ('  -c  <file>     configuration file')
437
    print ('  -d, --debug    debug')
438
439
440
441
442
443
444
445
446
447
    print ('  -h, --help     display this help and exit')
    print ('  -v, --verbose  verbose output')
    print ('  -V, --version  output version information and exit')
    print ('      --license  output license information and exit')

###
# License
###
def license():
    print('Copyright (C) 2014  Jason Frisvold <friz@godshell.com>');
Jason Frisvold's avatar
Jason Frisvold committed
448
449
    print('License GPLv2+: GNU GPL version 2 or later ' \
          '<http://gnu.org/licenses/gpl.html>.');
450
451
452
453
454
455
456
457
458
459
460
    print('This is free software: you are free to change and redistribute it.');
    print('There is NO WARRANTY, to the extent permitted by law.');
    print('');
    print('Written by Jason Frisvold.');

###
# Run the main program, pass in the CLI arguments (minus the command name)
try:
    if __name__ == "__main__":
        main(sys.argv[1:])
except KeyboardInterrupt:
461
    sys.exit()