Commit 1ff9a9b8 authored by toaster's avatar toaster

1.6beta2 checkin, report fixed, irods partial audit patch, sample python code as well

git-svn-id: f1b3a171-7291-4a19-a512-95ad0ad9394a
parent 1aa138ab
......@@ -21,7 +21,7 @@ In this file you should find the following:
use aceam;
source ace-am.sql;
----- Example -----
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 2 to server version: 4.1.20
# ACE Token store tool
import hashlib
import binascii
import string
import getopt
import sys
import time
from urlparse import urlparse
from suds.client import Client
from import DateTime
class TokenStore:
"""Ace Token Store"""
def __init__(self,infile,trimid=0):
self.entries = {}
self.roundlist = {}
while 1:
header = readHeader(infile)
if not header:
albName = header[0]
identifiers = readIdentifiers(infile)
proof = readProof(infile)
entry = TokenStoreEntry(proof,header)
for id in identifiers:
self.entries[id[trimid:]] = entry
def get_round(self,round):
if round not in self.roundlist:
client = Client(url)
response = client.service.getRoundSummaries(round)
for resp in response:
self.roundlist[] = resp.hashValue;
return self.roundlist[round]
def validate(self,file,identifier):
if identifier not in self.entries:
return None
token = self.entries[identifier]
roundhash = self.get_round(token.round)
prevhash = digestFile(file)
for proofLine in token.proof:
prevhash = calculateLevel(prevhash,proofLine,token.algorithm)
if (binascii.b2a_hex(prevhash) != roundhash):
return False
return True
class TokenStoreEntry:
"""Token Store Entry"""
def __init__(self,proof,headerparts):
self.proof = proof
self.algorithm,self.server,self.service,self.round,,self.length = headerparts
self.round = int(self.round)
def createTokens(digestlist,outfile,wsdl=''):
print 'start_create ' + str(time.time())
urlparts = urlparse(wsdl)
client = Client(wsdl)
requestlist = []
for digestpair in digestlist:
request = client.factory.create('tokenRequest')
request.hashValue = digestpair[1] = digestpair[0]
#request._tokenClassName = "SHA-256-0"
print 'before call' + str(time.time())
response = client.service.requestTokensImmediate('SHA-256-0',requestlist)
#response = portType.requestTokensImmediate(request)
print 'after call ' + str(time.time())
for item in response:
lines = [,'']
for proofelement in item.proofElements:
lines.append( ":".join(proofelement.hashes))
result = '\n'.join(lines)
outfile.write(item.digestService + ' ' + urlparts.hostname + ' ' + item.tokenClassName + ' ' + str(item.roundId) + ' ' +DateTime(item.timestamp).__unicode__()+ ' ' + str(len(result)) + "\n")
print 'return ' + str(time.time())
def getAlgorithm(algName):
if (algName == "SHA-256"):
return hashlib.sha256()
elif (algName == "SHA-512"):
return hashlib.sha512()
elif (algName == "SHA-384"):
return hashlib.sha384()
elif (algName == "MD5"):
return hashlib.md5()
elif (algName == "SHA1"):
return hashlib.sha1()
return None
def calculateLevel(lowerHash,rowString, algName):
"""Calculate a level given a token store string, and the hash and index
of the previously calculated level's (or file) hash"""
hashAlg = getAlgorithm(algName)
for hash in string.split(rowString,":"):
if (hash == "X"):
return hashAlg.digest()
def readHeader(file):
currLine = file.readline()
if not currLine:
return False
headerParts = string.split(currLine)
if (len(headerParts) != 6):
print "Bad header: " + currLine
return False
return headerParts
def readIdentifiers(infile):
line = infile.readline().rstrip("\n")
ids = []
while line != "":
line = infile.readline().rstrip("\n")
return ids
def readProof(infile):
line = infile.readline().rstrip("\n")
proof = []
while line != "":
line = infile.readline().rstrip("\n")
return proof
def digestFile(file,alg="SHA-256"):
hashAlg = getAlgorithm(alg)
with open(file,'rb') as digFile:
bytes_read =*1024)
while bytes_read:
bytes_read =*1024)
fileDigest = hashAlg.digest()
return fileDigest
import acestore
import os
import sys
import binascii
import argparse
class TrollSettings:
outfile = sys.stdout
requestlist = []
def processDir(settings,directory,files):
for f in files:
fullPath = os.path.join(directory,f)
if os.path.isfile(fullPath):
if len(settings.requestlist) > 1000:
def main():
parser = argparse.ArgumentParser(description='Validate Files using a token store.')
parser.add_argument('files',nargs='+',help='Files or directories to scan');
parser.add_argument('-r','--recurse',action='store_true',help='recurse into any listed directories')
parser.add_argument('-f','--file',nargs='?', type=argparse.FileType('w'), default=sys.stdout,help='File to write token store into, default std out')
parser.add_argument('-d','--digest',nargs='?',default='SHA-256',help='Digest algorithm to use (default SHA-256)')
args = parser.parse_args()
if acestore.getAlgorithm(args.digest) is None:
print 'Invalid digest algorithm ' + args.digest
settings = TrollSettings()
settings.outfile = args.file
for file in args.files:
if not (os.path.isfile(file) or args.recurse and os.path.isdir(file)):
if not args.recurse and os.path.isdir(file):
print file + " is a directory and -r, or --recurse has not been specified"
elif not os.path.isfile(file):
print file + " does not exist"
for file in args.files:
if os.path.isdir(file):
os.path.walk(file,processDir, settings)
if len(settings.requestlist) > 0:
if __name__ == "__main__":
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"Contributed modules"
# -*- coding: latin-1 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation; either version 3, or (at your option) any later
# version.
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTIBILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
"Pythonic simple SOAP Client implementation"
__author__ = "Mariano Reingart ("
__copyright__ = "Copyright (C) 2008 Mariano Reingart"
__license__ = "LGPL 3.0"
__version__ = "1.02c"
import urllib
import httplib2
Http = httplib2.Http
except ImportError:
import urllib2
class Http(): # wrapper to use when httplib2 not available
def request(self, url, method, body, headers):
f = urllib2.urlopen(urllib2.Request(url, body, headers))
from simplexml import SimpleXMLElement, TYPE_MAP, OrderedDict
class SoapFault(RuntimeError):
def __init__(self,faultcode,faultstring):
self.faultcode = faultcode
self.faultstring = faultstring
# soap protocol specification & namespace
soap_namespaces = dict(
class SoapClient(object):
"Simple SOAP Client (smil PHP)"
def __init__(self, location = None, action = None, namespace = None,
cert = None, trace = False, exceptions = True, proxy = None, ns=False,
soap_ns=None, wsdl = None, cache = False):
self.certssl = cert
self.keyssl = None
self.location = location # server location (url)
self.action = action # SOAP base action
self.namespace = namespace # message
self.trace = trace # show debug messages
self.exceptions = exceptions # lanzar execpiones? (Soap Faults)
self.xml_request = self.xml_response = ''
if not soap_ns and not ns:
self.__soap_ns = 'soap' # 1.1
elif not soap_ns and ns:
self.__soap_ns = 'soapenv' # 1.2
self.__soap_ns = soap_ns
# parse wsdl url = wsdl and self.wsdl(wsdl, debug=trace, cache=cache)
self.service_port = None # service port for late binding
if not proxy:
self.http = Http()
import socks
self.http = httplib2.Http(proxy_info = httplib2.ProxyInfo(
proxy_type=socks.PROXY_TYPE_HTTP, **proxy))
#if self.certssl: # esto funciona para validar al server?
# self.http.add_certificate(self.keyssl, self.keyssl, self.certssl)
self.__ns = ns # namespace prefix or False to not use it
if not ns:
self.__xml = """<?xml version="1.0" encoding="UTF-8"?>
<%(soap_ns)s:Envelope xmlns:xsi=""
<%(method)s xmlns="%(namespace)s">
self.__xml = """<?xml version="1.0" encoding="UTF-8"?>
<%(soap_ns)s:Envelope xmlns:%(soap_ns)s="%(soap_uri)s" xmlns:%(ns)s="%(namespace)s">
def __getattr__(self, attr):
"Return a pseudo-method that can be called"
if not # not using WSDL?
return lambda self=self, *args, **kwargs:,*args,**kwargs)
else: # using WSDL:
return lambda self=self, *args, **kwargs: self.wsdl_call(attr,*args,**kwargs)
def call(self, method, *args, **kwargs):
"Prepare xml request and make SOAP call, returning a SimpleXMLElement"
#TODO: method != input_message
# Basic SOAP request:
xml = self.__xml % dict(method=method, namespace=self.namespace, ns=self.__ns,
soap_ns=self.__soap_ns, soap_uri=soap_namespaces[self.__soap_ns])
request = SimpleXMLElement(xml,namespace=self.__ns and self.namespace, prefix=self.__ns)
# serialize parameters
if kwargs:
parameters = kwargs.items()
parameters = args
if parameters and isinstance(parameters[0], SimpleXMLElement):
# merge xmlelement parameter ("raw" - already marshalled)
for param in parameters[0].children():
# marshall parameters:
for k,v in parameters: # dict: tag=valor
self.xml_request = request.as_xml()
self.xml_response = self.send(method, self.xml_request)
response = SimpleXMLElement(self.xml_response, namespace=self.namespace)
if self.exceptions and response("Fault", ns=soap_namespaces.values(), error=False):
raise SoapFault(unicode(response.faultcode), unicode(response.faultstring))
return response
def send(self, method, xml):
"Send SOAP request using HTTP"
if self.location == 'test': return
location = "%s" % self.location #?op=%s" % (self.location, method)
soap_action = self.action
soap_action = self.action+method
'Content-type': 'text/xml; charset="UTF-8"',
'Content-length': str(len(xml)),
"SOAPAction": "\"%s\"" % (soap_action)
if self.trace:
print "-"*80
print "POST %s" % location
print '\n'.join(["%s: %s" % (k,v) for k,v in headers.items()])
print u"\n%s" % xml.decode("utf8","ignore")
response, content = self.http.request(
location,"POST", body=xml, headers=headers )
self.response = response
self.content = content
if self.trace:
print '\n'.join(["%s: %s" % (k,v) for k,v in response.items()])
print content#.decode("utf8","ignore")
print "="*80
return content
def get_operation(self, method):
# try to find operation in wsdl file
soap_ver = self.__soap_ns == 'soap12' and 'soap12' or 'soap11'
if not self.service_port:
for service_name, service in
for port_name, port in [port for port in service['ports'].items()]:
if port['soap_ver'] == soap_ver:
self.service_port = service_name, port_name
raise RuntimeError("Cannot determine service in WSDL: "
"SOAP version: %s" % soap_ver)
port =[self.service_port[0]]['ports'][self.service_port[1]]
self.location = port['location']
operation = port['operations'].get(unicode(method))
if not operation:
raise RuntimeError("Operation %s not found in WSDL: "
"Service/Port Type: %s" %
(method, self.service_port))
return operation
def wsdl_call(self, method, *args, **kwargs):
"Pre and post process SOAP call, input and output parameters using WSDL"
soap_uri = soap_namespaces[self.__soap_ns]
operation = self.get_operation(method)
# get i/o type declarations:
input = operation['input']
output = operation['output']
if 'action' in operation:
self.action = operation['action']
# sort parameters (same order as xsd:sequence)
def sort_dict(od, d):
if isinstance(od, dict):
ret = OrderedDict()
for k in od.keys():
v = d.get(k)
if v:
if isinstance(v, dict):
v = sort_dict(od[k], v)
elif isinstance(v, list):
v = [sort_dict(od[k][0], v1)
for v1 in v]
ret[str(k)] = v
return ret
return d
if input and kwargs:
params = sort_dict(input.values()[0], kwargs).items()
method = input.keys()[0]
#elif not input:
#TODO: no message! (see wsmtxca.dummy)
params = kwargs and kwargs.items()
# call remote procedure
response =, *params)
# parse results:
resp = response('Body',ns=soap_uri).children().unmarshall(output)
return resp and resp.values()[0] # pass Response tag children
def help(self, method):
"Return operation documentation and invocation/returned value example"
operation = self.get_operation(method)
input = operation['input'].values()
input = input and input[0]
output = operation['output'].values()[0]
return u"%s(%s)\n -> %s:\n\n%s" % (
input and ", ".join("%s=%s" % (k,repr(v)) for k,v
in input.items()) or "",
output and output or "",
def wsdl(self, url, debug=False, cache=False):
"Parse Web Service Description v1.1"
soap_ns = {
"": 'soap11',
"": 'soap12',
get_local_name = lambda s: str((':' in s) and s.split(':')[1] or s)
REVERSE_TYPE_MAP = dict([(v,k) for k,v in TYPE_MAP.items()])
def fetch(url):
"Fetch a document from a URL, save it locally if cache enabled"
import os, hashlib
# make md5 hash of the url for caching...
filename = "%s.xml" % hashlib.md5(url).hexdigest()
if isinstance(cache, basestring):
filename = os.path.join(cache, filename)
if cache and os.path.exists(filename):
if debug: print "Reading file %s" % (filename, )
f = open(filename, "r")
xml =
if debug: print "Fetching url %s" % (url, )
f = urllib.urlopen(url)
xml =
if cache:
if debug: print "Writing file %s" % (filename, )
f = open(filename, "w")
return xml
# Open uri and read xml:
xml = fetch(url)
# Parse WSDL XML:
wsdl = SimpleXMLElement(xml, namespace=wsdl_uri)
# detect soap prefix and uri (xmlns attributes of <definitions>)
xsd_ns = None
soap_uris = {}
for k, v in wsdl[:]:
if v in soap_ns and k.startswith("xmlns:"):
soap_uris[get_local_name(k)] = v
if v== xsd_uri and k.startswith("xmlns:"):
xsd_ns = get_local_name(k)
# Extract useful data:
self.namespace = wsdl['targetNamespace']
self.documentation = unicode(wsdl('documentation', error=False) or '')
services = {}
bindings = {} # binding_name: binding
operations = {} # operation_name: operation
port_type_bindings = {} # port_type_name: binding
messages = {} # message: element
elements = {} # element: type def
for service in wsdl.service:
if not service_name:
continue # empty service?
if debug: print "Processing service", service_name
serv = services.setdefault(service_name, {'ports': {}})
serv['documentation']=service['documentation'] or ''
for port in service.port:
binding_name = get_local_name(port['binding'])
address = port('address', ns=soap_uris.values(), error=False)
location = address and address['location'] or None
soap_uri = address and soap_uris.get(address.get_prefix())
soap_ver = soap_uri and soap_ns.get(soap_uri)
bindings[binding_name] = {'service_name': service_name,
'location': location,
'soap_uri': soap_uri, 'soap_ver': soap_ver,
serv['ports'][port['name']] = bindings[binding_name]
for binding in wsdl.binding:
binding_name = binding['name']
if debug: print "Processing binding", service_name
soap_binding = binding('binding', ns=soap_uris.values(), error=False)
transport = soap_binding and soap_binding['transport'] or None
port_type_name = get_local_name(binding['type'])
'port_type_name': port_type_name,
'transport': transport, 'operations': {},
port_type_bindings[port_type_name] = bindings[binding_name]
for operation in binding.operation:
op_name = operation['name']
op = operation('operation',ns=soap_uris.values(), error=False)
action = op and op['soapAction']
d = operations.setdefault(op_name, {})
bindings[binding_name]['operations'][op_name] = d
d.update({'name': op_name})
#if action: #TODO: separe operation_binding from operation
if action:
d["action"] = action
#TODO: cleanup element/schema/types parsing:
def process_element(element_name, node):
"Parse and define simple element types"
if debug: print "Processing element", element_name
for tag in node:
if tag.get_local_name() in ("annotation", "documentation"):
elif tag.get_local_name() in ('element', 'restriction'):
if debug: print element_name,"has not children!",tag
children = tag # element "alias"?
alias = True
elif tag.children():
children = tag.children()
alias = False
if debug: print element_name,"has not children!",tag
continue #TODO: abstract?
d = OrderedDict()
for e in children:
t = e['type']
if not t:
t = e['base'] # complexContent (extension)!
if not t:
t = 'anyType' # no type given!
t = t.split(":")
if len(t)>1:
ns, type_name = t
ns, type_name = None, t[0]
if element_name == type_name:
continue # prevent infinite recursion
uri = ns and e.get_namespace_uri(ns) or xsd_uri
if uri==xsd_uri:
# look for the type, None == any
fn = REVERSE_TYPE_MAP.get(unicode(type_name), None)
# complex type, postprocess later
fn = elements.setdefault(unicode(type_name), OrderedDict())
if e['name'] is not None and not alias:
e_name = unicode(e['name'])
d[e_name] = fn
if debug: print "complexConent/simpleType/element", element_name, "=", type_name
d[None] = fn
if e['maxOccurs']=="unbounded":
# it's an array... TODO: compound arrays?
d.array = True
if e is not None and e.get_local_name() == 'extension' and e.children():
# extend base element:
process_element(element_name, e.children())
elements.setdefault(element_name, OrderedDict()).update(d)
# check axis2 namespace at schema types attributes
self.namespace = dict(wsdl.types("schema", ns=xsd_uri)[:]).get('targetNamespace', self.namespace)
imported_schemas = {}
def preprocess_schema(schema):
"Find schema elements and complex types"
for element in schema.children():
if element.get_local_name() in ('import', ):
schema_namespace = element['namespace']
schema_location = element['schemaLocation']
if schema_location is None:
if debug: print "Schema location not provided for %s!" % (schema_namespace, )
if schema_location in imported_schemas:
if debug: print "Schema %s already imported!" % (schema_location, )
imported_schemas[schema_location] = schema_namespace
if debug: print "Importing schema %s from %s" % (schema_namespace, schema_location)
# Open uri and read xml:
xml = fetch(schema_location)
# Parse imported XML schema (recursively):