Moved Docker stuff to "Docker" folder

Created k8s folder for k8s stuff
Added early-stage service.yaml for K8s deployment
This commit is contained in:
erichardso
2018-08-28 11:51:23 -07:00
parent 176e2f2062
commit d880f44ca6
138 changed files with 11 additions and 0 deletions

View File

@@ -0,0 +1,87 @@
#=============================================================================
import StringIO
#=============================================================================
class Data:
def __init__( self, dict=None, **kwargs ):
if dict:
self.__dict__.update( dict )
if kwargs:
self.__dict__.update( kwargs )
#--------------------------------------------------------------------------
def keys( self ):
return self.__dict__.keys()
#--------------------------------------------------------------------------
def update( self, rhs ):
return self.__dict__.update( rhs.__dict__ )
#--------------------------------------------------------------------------
def __setitem__( self, key, value ):
self.__dict__[key] = value
#--------------------------------------------------------------------------
def __getitem__( self, key ):
return self.__dict__[key]
#--------------------------------------------------------------------------
def __contains__( self, key ):
return key in self.__dict__
#--------------------------------------------------------------------------
def __str__( self ):
out = StringIO.StringIO()
self._formatValue( self, out, 3 )
return out.getvalue()
#--------------------------------------------------------------------------
def __repr__( self ):
return self.__str__()
#--------------------------------------------------------------------------
def _formatValue( self, value, out, indent ):
if isinstance( value, Data ):
out.write( "%s(\n" % self.__class__.__name__ )
for k, v in sorted( value.__dict__.iteritems() ):
if k[0] == "_":
continue
out.write( "%*s%s" % ( indent, '', k ) )
out.write( " = " )
self._formatValue( v, out, indent+3 )
out.write( ",\n" )
out.write( "%*s)" % ( indent, '' ) )
elif isinstance( value, dict ):
out.write( "{\n" )
for k, v in sorted( value.iteritems() ):
if k[0] == "_":
continue
out.write( "%*s" % ( indent, '' ) )
self._formatValue( k, out, 0 )
out.write( " : " )
self._formatValue( v, out, indent+3 )
out.write( ",\n" )
out.write( "%*s}" % ( indent, '' ) )
elif isinstance( value, list ):
out.write( "[\n" )
for i in value:
out.write( "%*s" % ( indent, '' ) )
self._formatValue( i, out, indent+3 )
out.write( ",\n" )
out.write( "%*s]" % ( indent, '' ) )
elif isinstance( value, str ):
out.write( "'%s'" % ( value ) )
else:
out.write( "%s" % ( value ) )
#=============================================================================

View File

@@ -0,0 +1,61 @@
#===========================================================================
#
# Error
#
#===========================================================================
""": Stack based error message exception class"""
#===========================================================================
import sys
#===========================================================================
class Error ( Exception ):
""": Stack based error message exception class.
"""
#-----------------------------------------------------------------------
@staticmethod
def raiseException( exception, msg ):
excType, excValue, trace = sys.exc_info()
if not isinstance( exception, Error ):
exception = Error( str( exception ) )
exception.add( msg )
raise exception, None, trace
#-----------------------------------------------------------------------
@staticmethod
def fromException( exception, msg ):
excType, excValue, trace = sys.exc_info()
newError = Error( str( exception ) )
newError.add( msg )
raise newError, None, trace
#-----------------------------------------------------------------------
def __init__( self, msg ):
""": Constructor
"""
self._msg = [ msg ]
Exception.__init__( self )
#-----------------------------------------------------------------------
def add( self, msg ):
self._msg.append( msg )
#-----------------------------------------------------------------------
def __str__( self ):
s = "\n"
for msg in reversed( self._msg ):
s += "- %s\n" % msg
return s
#-----------------------------------------------------------------------
#===========================================================================

View File

@@ -0,0 +1,51 @@
#===========================================================================
#
# Named structure field class
#
#===========================================================================
import struct
from .Data import Data
#==============================================================================
class NamedStruct:
#---------------------------------------------------------------------------
def __init__( self, endian, elems ):
"""Constructr
endian == BIG_ENDIAN or LITTLE_ENDIAN
elems = [ ( struct_format_code, name ), ... ]
"""
assert( endian == "BIG_ENDIAN" or endian == "LITTLE_ENDIAN" )
if endian == "BIG_ENDIAN":
self.format = ">"
elif endian == "LITTLE_ENDIAN":
self.format = "<"
self.format += "".join( [ i[0] for i in elems ] )
self.names = [ i[1] for i in elems ]
self.struct = struct.Struct( self.format )
#---------------------------------------------------------------------------
def __len__( self ):
return self.struct.size
#---------------------------------------------------------------------------
def pack( self, obj ):
data = [ getattr( obj, i ) for i in self.names ]
return self.struct.pack( *data )
#---------------------------------------------------------------------------
def unpack( self, obj, bytes, offset=0 ):
if obj is None:
obj = Data()
data = self.struct.unpack_from( bytes, offset )
for i in range( len( self.names ) ):
setattr( obj, self.names[i], data[i] )
return obj
#==============================================================================

View File

@@ -0,0 +1,19 @@
#=============================================================================
#
# General utilities
#
#=============================================================================
from . import config
from .Data import Data
from .Error import Error
from .fimport import fimport
from . import hex
from . import jsonUtil as json
from . import log
from .NamedStruct import NamedStruct
from . import path
from . import process
from . import test
#=============================================================================

View File

@@ -0,0 +1,68 @@
#===========================================================================
#
# Config file utilities.
#
#===========================================================================
__doc__ = """Config file utilities.
"""
from . import path
from . import fimport
#===========================================================================
class Entry:
def __init__( self, name, cvt, default=None ):
self.name = name
self.cvt = cvt
self.default = default
#===========================================================================
def readAndCheck( configDir, configFile, entries ):
# Combine the dir and file, expand any variables, and read the
# python code into a module.
configPath = path.expand( configDir, configFile )
m = fimport.fimport( configPath )
# Check the input values for the correc types and assign any
# default values.
check( m, entries )
return m
#===========================================================================
def check( input, entries ):
if isinstance( input, dict ):
checkDict( input, entries )
return
# Use the sections to do error checking.
for e in entries:
if not hasattr( input, e.name ):
value = e.default
# Run the converter function on the input. This validates the
# input type and can do any other manipulations it wants.
elif e.cvt:
inputValue = getattr( input, e.name )
value = e.cvt( inputValue )
setattr( input, e.name, value )
#===========================================================================
def checkDict( input, entries ):
assert( isinstance( input, dict ) )
# Use the sections to do error checking.
for e in entries:
if not e.name in input:
value = e.default
# Run the converter function on the input. This validates the
# input type and can do any other manipulations it wants.
elif e.cvt:
inputValue = input[e.name]
value = e.cvt( inputValue )
input[e.name] = value
#===========================================================================

View File

@@ -0,0 +1,28 @@
#===========================================================================
#
# Arbitrary file importing utility. Does NOT modify sys.modules
#
#===========================================================================
import imp
import os
def fimport( filePath ):
# Read the file and compile the code. This will fail if the file
# doesn't exist or there are problems w/ the syntax in the file.
with open( filePath, 'r' ) as f:
code = compile( f.read(), filePath, "exec", dont_inherit=True )
# Get the absolute path and the file name w/o the directory or
# extension to set into the module variables.
absPath = os.path.abspath( filePath )
d, fileName = os.path.split( filePath )
rootName, ext = os.path.splitext( fileName )
# Create a new module and exec the code in it's context.
m = imp.new_module( rootName )
m.__file__ = absPath
exec code in m.__dict__
# Return the module object.
return m

View File

@@ -0,0 +1,10 @@
#=============================================================================
#
# Hex string/byte utilities
#
#=============================================================================
from .dump import dump
from .toBytes import toBytes
#=============================================================================

View File

@@ -0,0 +1,32 @@
#===========================================================================
#
# Dump hex bytes to a table.
#
#===========================================================================
import StringIO
#===========================================================================
def dump( buf ):
"""Input is bytes buffer,
Returns a string w/ the hex values in a table
"""
# Convert to hex characters
h = [ i.encode( "hex" ).upper() for i in buf ]
f = StringIO.StringIO()
f.write( "---: 00 01 02 03 04 05 06 07 08 09\n" )
for i in range( len( h ) ):
if i % 10 == 0:
if i > 0:
f.write( "\n" )
f.write( "%03d: " % i )
f.write( "%2s " % h[i] )
f.write( "\n" )
return f.getvalue()
#===========================================================================

View File

@@ -0,0 +1,23 @@
#===========================================================================
#
# Convert a hex string to bytes
#
#===========================================================================
import math
#===========================================================================
def toBytes( hexStr ):
"""Input is a string containing hex values (w/ or w/o spaces)
Return is the same value in a bytes array.
"""
s = hexStr.strip().replace( "\n", " " )
s = ''.join( s.split(" ") )
bytes = []
for i in range( 0, len( s ), 2 ):
bytes.append( chr( int( s[i:i+2], 16 ) ) )
return ''.join( bytes )
#===========================================================================

View File

@@ -0,0 +1,54 @@
#=============================================================================
#
# JSON utility that turns unicode strings to ascii
#
# NOTE: this file should be named json.py but Python's stupid import
# rules look in the current directory first instead of using absolute
# paths all the time. So we can't import the global json module if we
# do that.
#
# Code from:
# http://stackoverflow.com/questions/956867/how-to-get-string-objects-instead-of-unicode-ones-from-json-in-python
#
#=============================================================================
import json
#=============================================================================
# For completeness, add the save API's
dump = json.dump
dumps = json.dumps
#=============================================================================
def load( file ):
"""Same as json.load() but turns unicode to ascii strings.
"""
return _toStr( json.load( file, object_hook=_toStr ), ignoreDicts=True )
#=============================================================================
def loads( text ):
"""Same as json.loads() but turns unicode to ascii strings.
"""
return _toStr( json.loads( text, object_hook=_toStr ), ignoreDicts=True )
#=============================================================================
def _toStr( data, ignoreDicts=False ):
# Convert unicode to string.
if isinstance( data, unicode ):
return data.encode( 'utf-8' )
# For lists, process each item.
if isinstance( data, list ):
return [ _toStr( i, ignoreDicts=True ) for i in data ]
# For dicts, process keys and values, but only if we haven't
# already byteified it
if isinstance( data, dict ) and not ignoreDicts:
return {
_toStr( k, ignoreDicts=True ) : _toStr( v, ignoreDicts=True )
for k, v in data.iteritems()
}
# Otherwise return the original object.
return data
#=============================================================================

View File

@@ -0,0 +1,82 @@
#=============================================================================
#
# Logging utilities
#
#=============================================================================
import logging
import logging.handlers
import platform
import sys
import types
from .Error import Error
from . import path
#=============================================================================
_logs = {}
#=============================================================================
def get( name, level=None, output=None ):
""" TODO: doc
"""
log = _logs.get( name, None )
if not log:
log = create( name )
if level is not None:
log.setLevel( level )
if output is not None:
writeTo( log, output )
return log
#=============================================================================
def create( name, level=logging.ERROR ):
""" Create a logging object for the package name.
The full logging name will be 'tHome.NAME'.
"""
log = logging.getLogger( 'tHome.%s' % name )
log.setLevel( level )
handler = logging.NullHandler()
handler.setFormatter( _formatter() )
log.addHandler( handler )
# Monkey patch a method onto the log class.
log.writeTo = types.MethodType( writeTo, log )
# Save a handle to the log.
_logs[name] = log
return log
#=============================================================================
def writeTo( log, fileName ):
""" TODO: doc
"""
if fileName == "stdout":
handler = logging.StreamHandler( sys.stdout )
else:
try:
path.makeDirs( fileName )
# Use the watcher on linux so that logrotate will work
# properly. It will close and reopen the log file if the OS
# moves it. Not supported on windows.
if platform.system() == "Windows":
handler = logging.FileHandler( fileName )
else:
handler = logging.handlers.WatchedFileHandler( fileName )
except ( Error, IOError ) as e:
msg = "Error trying to open the log file '%s' for writing." % fileName
Error.raiseException( e, msg )
handler.setFormatter( _formatter() )
log.addHandler( handler )
#=============================================================================
def _formatter():
return logging.Formatter( '%(asctime)s : %(levelname)s: %(message)s' )
#=============================================================================

View File

@@ -0,0 +1,229 @@
#===========================================================================
#
# Poll
#
#===========================================================================
import select
import errno
#===========================================================================
class Poll:
""": Posix poll manager.
This classes manages the polling behavior for normal posix (non-gui)
network code. See the Python select module or UNIX poll
documentation for details.
Server and Link objects are added to the Poll and it will manage
the I/O calls for each of them. Instead of having a poll() method
like the Python module, this class has a select() method to kee the
API identical to the mpy.net.Select class.
= EXAMPLE
# MPY_NO_TEST
# # Setup server and/or links here.
#
# # Create the select and add the servers/links to it.
# mgr = Poll()
# for c in clients:
# mgr.add( c )
#
# # Loop to process.
# while True:
# mgr.select()
"""
#-----------------------------------------------------------------------
def __init__( self ):
""": Constructor."""
# Create the polling object.
self.poll = select.poll()
# Key is a file descripter for the socket of the object being
# watched. Value is a Link or Server object for that socket.
self.clients = {}
# Bit flags to watch for when registering a socket for read or
# read/write.
self.READ = select.POLLIN | select.POLLPRI | select.POLLHUP | \
select.POLLERR
self.READ_WRITE = self.READ | select.POLLOUT
# Bit flags reported for events.
self.EVENT_READ = select.POLLIN | select.POLLPRI
self.EVENT_WRITE = select.POLLOUT
self.EVENT_CLOSE = select.POLLHUP
self.EVENT_ERROR = select.POLLERR
# NOTE: These bit flags were originally class variables
# (accessed via Poll.READ, etc) but a weird race condition in
# Python multi-threaded code (mpy.parallel) would sometimes
# throw an exception saying Poll was None. No idea why that
# could happen but making these member variables seems to fix
# the problem.
#-----------------------------------------------------------------------
def active( self ):
""": Return the number of sockets being watched.
Used for single point connections to automatically exit a select
function when connections close.
= EXAMPLE
# MPY_NO_TEST
# select = Poll()
# select.add( client1 )
# select.add( client12 )
#
# while select.active():
# select.select()
"""
return len( self.clients )
#-----------------------------------------------------------------------
def add( self, obj ):
""": Add a link or server to the select.
Call obj.close() to remove the object.
= INPUT VARIABLES
- obj The Link or Server object to add to the select.
"""
# All sockets are checked for reading since a read of 0 means a
# dropped connection. Errors are also checked even though they
# rarely occur. Sockets for writing is handled in the
# clientCanWrite() callback that is triggered through the
# watcher mechanism below.
self.poll.register( obj.fileno, self.READ )
# Save the object for later use.
self.clients[ obj.fileno ] = obj
# Tell the object it's part of this Poll. This allows it to
# close the connection and tell the Poll about it. In
# addition, the client can notify us if there is data to write
# or not.
obj.sigClosing.connect( self.closeLink )
obj.sigCanWrite.connect( self.linkCanWrite )
#-----------------------------------------------------------------------
def remove( self, obj ):
""": Remove an object from the select.
Normally this is called by the object being removed when
obj.close() is called (i.e. this should not normally be called).
= INPUT VARIABLES
- obj The object that is closing. This should be the
same object that was passed to add().
"""
# Remove the object from our structures.
self.poll.unregister( obj.fileno )
del self.clients[ obj.fileno ]
# Tell the object to remove ourselves from it's watch list.
obj.sigClosing.disconnect( self.closeLink )
obj.sigCanWrite.disconnect( self.linkCanWrite )
#-----------------------------------------------------------------------
def closeAll( self ):
""": Close all clients attached to the select.
This will close any Link and Server objects that are part of
the select.
"""
# As we close clients, they will notify the select to remove
# themselves. So make a copy of the dict value list list
# containing all the objects so we're not modifying the client
# dictionary as we loop over it.
clients = self.clients.values()[:]
for c in clients:
c.close()
#-----------------------------------------------------------------------
def select( self, timeOut=None ):
""": Watch the sockets.
This will call poll.poll() (normal posix poll) and process the
results when it returns. All the clients and servers in the
poll (see the add() method) are checked for reading or errors.
Any client that returns True when client.canWrite() is called
will be checked for writing.
If a client has an error, then client.close() is called. If a
client can read, then client.read() is called. If a client can
write, then client.write() is called.
See the class documentation for an example.
= INPUT VARIABLES
- timeOut Optional floating point time out for the select call
in seconds. See select.select for more information.
"""
timeOut_msec = None if timeOut is None else 1000*timeOut
while True:
try:
# Returns tuple (fileDescriptor, bitFlag) of the files
# that can act.
events = self.poll.poll( timeOut_msec )
except select.error, v:
# Not sure why, but sometimes with a timeout value, you can
# get an "interrupted system call" error thrown. Web
# searches indicate this isn't really an error and should be
# ignored so test for it here.
if v[0] != errno.EINTR:
raise
else:
break
for fd, flag in events:
# Find the object for this file descriptor. Certain
# multi-threaded test cases seem to have a problem with the
# clientClosing() method not being called (which unregisters
# the fd) so handle that with a None return here.
obj = self.clients.get( fd, None )
if obj is None:
continue
# Object has data to read. If an error occurred during the
# read, clear the flag so we don't try to do anything else.
if flag & self.EVENT_READ:
if obj.readFromSocket() == -1:
flag = 0
# Object can write data.
if flag & self.EVENT_WRITE:
obj.writeToSocket()
# Test for errors or closing connections. Only call close once.
# Connection going down.
if flag & self.EVENT_CLOSE:
obj.close()
# Error - close the connection - use else here
elif flag & self.EVENT_ERROR:
obj.close()
#-----------------------------------------------------------------------
def linkCanWrite( self, obj, writeActive ):
# Writing should be active.
if writeActive:
# Add the socket if it's not already in the write list.
self.poll.modify( obj.fileno, self.READ_WRITE )
# Writing should not be active:
else:
# Remove the socket if it's in the write list.
self.poll.modify( obj.fileno, self.READ )
#-----------------------------------------------------------------------
def closeLink( self, obj ):
self.remove( obj )
#-----------------------------------------------------------------------
#===========================================================================

View File

@@ -0,0 +1,44 @@
#===========================================================================
#
# File and directory utilities.
#
#===========================================================================
import os
import os.path
from .Error import Error
#===========================================================================
def makeDirs( fileName ):
"""TODO: docs
"""
try:
d = os.path.dirname( fileName )
if d and not os.path.exists( d ):
os.makedirs( d )
except ( IOError, OSError ) as e:
msg = "Error trying to create intermediate directories for the file: " \
"'%s'" % ( fileName )
Error.raiseException( e, msg )
#===========================================================================
def expand( filePath, fileName=None ):
"""Combine a directory and file name and expand env variables and ~.
A full path can be input in filePath. Or a directory can be input
in filePath and a file name input in fileName.
"""
if fileName:
filePath = os.path.join( filePath, fileName )
filePath = str( filePath )
if "$" in filePath:
filePath = os.path.expandvars( filePath )
if "~" in filePath:
filePath = os.path.expanduser( filePath )
return filePath
#===========================================================================

View File

@@ -0,0 +1,5 @@
#=============================================================================
from .simple import simple
#=============================================================================

View File

@@ -0,0 +1,22 @@
#=============================================================================
import subprocess
#=============================================================================
def simple( cmd, cwd=None ):
"""Runs a command and returns stdout and stderr mixed together.
Throws an Exception w/ the output if it fails.
"""
# Set stderr to also send output to stdout (i.e. combine them).
p = subprocess.Popen( cmd, cwd=cwd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT )
( stdout, stderr ) = p.communicate()
if p.returncode:
raise Exception( stdout )
return stdout
#=============================================================================

View File

@@ -0,0 +1,30 @@
#=============================================================================
#
# Testing utilites.
#
#=============================================================================
import unittest
#=============================================================================
class Case( unittest.TestCase ):
def __init__( self, cls ):
unittest.TestCase.__init__( self, cls )
def setup( self ):
pass
def teardown( self ):
pass
def eq( self, a, b, msg=None ):
if not a == b:
msg = msg if msg else ""
#self._errors.append( "%s %r != %r" % ( msg, a, b ) )
self.fail( "%s %r != %r" % ( msg, a, b ) )
def neq( self, a, b, msg=None ):
if not a != b:
msg = msg if msg else ""
#self._errors.append( "%s %r == %r" % ( msg, a, b ) )
self.fail( "%s %r == %r" % ( msg, a, b ) )

View File

@@ -0,0 +1,19 @@
from tHome.util import Data
d = Data( a=1, b="asdf", c=2 )
print d
print "----"
d = Data( a=1, b="asdf", c=[2,3,4] )
print d
print "----"
d = Data( a=1, b="asdf", c={'a':3, 'b':4} )
print d
print "----"
d = Data( a=1, b=[ Data(a=1,b=2) ], c={'a':3, 'b':[1,2,3]} )
print d
print "----"