- Added code to crawl config directory and produce list of files to process

- Added sqlite code to create database
- Updated design document

client/spawner.default.conf

+#* Skynet - Automated "Cloud" Security Scanner                                *#
+#* Copyright (C) 2014  Jason Frisvold <friz@godshell.com>                     *#
+#*                                                                            *#
+#* This program is free software; you can redistribute it and/or modify       *#
+#* it under the terms of the GNU General Public License as published by       *#
+#* the Free Software Foundation; either version 2 of the License, or          *#
+#* (at your option) any later version.                                        *#
+#*                                                                            *#
+#* This program is distributed in the hope that it will be useful,            *#
+#* but WITHOUT ANY WARRANTY; without even the implied warranty of             *#
+#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *#
+#* GNU General Public License for more details.                               *#
+#*                                                                            *#
+#* You should have received a copy of the GNU General Public License          *#
+#* along with this program; if not, write to the Free Software                *#
+#* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA *#
+
+# This file defines the default values used by the spawner
+
+# DO NOT DELETE OR ALTER THIS FILE
+
+[spawner]
+
+# The configuration directory is where new instructions are placed by the server
+# This can be an absolute or relative path.
+configdir = config
+
+# The data directory is where the spawner will store internal data.
+datadir = spawner_data
+
+# This defines the name of the sqlite database used for timing data.  It will be
+# located in the datadir.
+timingdb = timing.db

client/spawner.py

@@ -25,6 +25,8 @@ import time
 import sqlite3
 import logging
 from datetime import datetime
+import os, os.path
+import re
 
 # Global Variables
 verbose = False
@@ -74,12 +76,24 @@ def main(argv):
 
     loadconfig()
 
+    # Check the config dir for new files
+    # Process any files found
+    
+    # Check timing table
+    # New process spawner
+    # Handle finished processes
+    
+    # Go to top of loop
+    
     # Main loop
     while True:
-        if check_configdir():
-            process_config()
+        timingdb = open_timing_database()
+
+        filelist = check_configdir()
+        if (len(filelist) > 0):
+            process_config(timingdb, filelist)
 
-        check_timing()
+        check_timing(timingdb)
         spawn_process()
         check_complete()
 
@@ -107,25 +121,31 @@ def loadconfig():
     logger.debug('datadir = {0}'.format(cfg['datadir']))
     logger.debug('timingdb = {0}'.format(cfg['timingdb']))
 
-    # Check the config dir for new files
-    # Process any files found
-    
-    # Check timing table
-    # New process spawner
-    # Handle finished processes
-    
-    # Go to top of loop
-    
 def check_configdir():
     logger.debug('Check Config')
     
-    # Check existence of sqlite file
-    # If doesn't exit, create
+    if (os.path.exists(cfg['configdir'])):
+        # Look for new files
+
+        filelist = []
 
-def process_config():
+        for name in os.listdir(cfg['configdir']):
+            if re.match('.*\.skynet$', name):
+                filelist.append(name)
+                print 'process file ', name
+
+        return filelist
+    else:
+        logger.critical('Configuration directory does not exist')
+        sys.exit()
+
+def process_config(timingdb, filelist):
     logger.debug('Process Config')
+    
+    for file in filelist:
+        print 'processing ', file
         
-def check_timing():
+def check_timing(timingdb):
     logger.debug('Check Timing')
 
 def spawn_process():
@@ -134,6 +154,42 @@ def spawn_process():
 def check_complete():
     logger.debug('Check for completed processes')
 
+def open_timing_database():
+    if (os.path.exists(cfg['datadir'])):
+        try:
+            timingdb = sqlite3.connect(os.path.join(cfg['datadir'], cfg['timingdb']))
+
+            timingcursor = timingdb.cursor()
+
+            # Create the tables, if they don't exist already
+            timingcursor.execute('''CREATE TABLE IF NOT EXISTS timers
+                                ( id                        INT,
+                                server_id                   INT,
+                                minute                      TEXT,
+                                hour                        TEXT,
+                                day                         TEXT,
+                                month                       TEXT,
+                                override_flag               BOOLEAN,
+                                ip_range                    TEXT,
+                                nmap_options                TEXT
+                                )''')
+
+            timingcursor.execute('''CREATE TABLE IF NOT EXISTS spawned
+                                ( id                        INT,
+                                start_time                  INT,
+                                pid                         INT
+                                )''')
+
+        except:
+            e = sys.exc_info()[0]
+            logger.exception(e)
+            sys.exit()
+
+        return(timingdb)
+    else:
+        logger.critical('Data Directory does not exist')
+        sys.exit()
+
 ###
 # Usage
 ###
@@ -144,7 +200,7 @@ def usage():
     print ('Mandatory arguments to long options are mandatory for short ' \
            'options too.')
     print ('  -c  <file>     configuration file')
-    print ('  -f             run in the foreground')
+    #print ('  -f             run in the foreground')
     print ('  -h, --help     display this help and exit')
     print ('  -v, --verbose  verbose output')
     print ('  -V, --version  output version information and exit')

doc/Skynet design document.txt

@@ -66,31 +66,31 @@ This is a “living” document and is not feature complete.  Check back often f
 
 The "cloud" piece of this software is the dumb workhorse piece of the system. Setup should be minimal and easily deployed on disparate systems.
 
-Instructions are delivered via flat text files placed into a configuration directory.  The local system parses these files and builds a localized timing table for spawning processes.  This localized data is stored in a flat text file with a predefined format.  Something similar to a cron table seems appropriate.  The parsing system should create a hash table of existing configuration files to identify what is new and what can be removed from the timing table. (Should this data be encrypted?  What advantage does this give an attacker?)
+Instructions are delivered via flat text files placed into a configuration directory.  The local system parses these files and builds a localized timing table for spawning processes.  This localized data is stored in a sqlite database.  The incoming configuration files identify what can be added and what can be removed from the timing table.  Incoming files are JSON encoded and contain the timing information, ip range, nmap options, override flags, and the id assigned by the server.
 
 A spawning daemon is responsible for reading the timing table and spawning new processes at the appropriate time.  New scans are spawned as separate processes with their PID being noted by the spawning daemon.  The spawning daemon should identify if the previous scan process has completed prior to starting a new process.  In the event of an existing process, the daemon should identify if the process is still running (PID file and PID exists) or if it died in process (PID file only).  It should send an appropriate notification to the administrator identifying the problem for dead PIDs and only send a notification for existing PIDs if an override flag is not set.  In the case of a dead PID, the new scan should be spawned as requested.  If the PID still exists, however, the spawning daemon should only spawn the process if there's an override flag set.  This gives the administrator control to run scans on a tighter schedule when the run-time of a single scan may exceed the period of time between scans.
 
 Finished scans should encrypt the scan results using a public GPG key and the plain text version of the file should be scrubbed.  (Can we encrypt on the fly as the scan is running?)  All completed files are stored in a holding area until the central processing system retrieves them.  After retrieval, reports are scrubbed from the system.
 
-Timing Table Format (sqlite3?)
+Timing Table Format
 
 CREATE TABLE timers (
-    id                          int,
-    minute
-    hour
-    day
-    month
-    override_flag
-    ip_range
-    nmap_options
+    id                          INT,
+    server_id                   INT,
+    minute                      TEXT,
+    hour                        TEXT,
+    day                         TEXT,
+    month                       TEXT,
+    override_flag               BOOLEAN,
+    ip_range                    TEXT,
+    nmap_options                TEXT
 )
 
 CREATE TABLE spawned (
-    id                          int,
-    start_time                  int,
-    pid                         int,
+    id                          INT,
+    start_time                  INT,
+    pid                         INT
 )
-minute hour day month override_flag ip_range nmap_options
 
 ### Control System