Commit ff20794f authored by shake's avatar shake
Browse files

Moved classes for the shell into separate lib. Added importing token stores to...

Moved classes for the shell into separate lib. Added importing token stores to duraspace, more verbose help, and updated code

git-svn-id: https://subversion.umiacs.umd.edu/ace/trunk@195 f1b3a171-7291-4a19-a512-95ad0ad9394a
parent e75b8579
#!/usr/bin/python2
#
# Classes which the ace shell depends on
#
import readline
import re
import os
class AceShell(object):
"""Ace Shell Information"""
def __init__(self, user=None, passwd=None, dhost=None, space=None,
wsdl='http://ims.umiacs.umd.edu:8080/ace-ims/IMSWebService?wsdl',
verbose=False):
self.user = user
self.passwd = passwd
self.wsdl = wsdl
self.dhost = dhost
self.spaceID = space
self.verbose = verbose
self.run = True
self.validate = True
self.commands = None
self.helper = None
self.completer = None
def printInfo(self):
print 'WSDL: ', self.wsdl, '\n',
print 'Duracloud Host: ', self.dhost, '\n',
print 'Duracloud SpaceID: ', self.spaceID, '\n',
print 'Duracloud Username: ', self.user, '\n',
print 'Validate on pull: ', self.validate, '\n',
print 'Verbose mode: ', self.verbose
def ready(self):
return (self.dhost!= None and self.spaceID != None and
self.user != None and self.passwd != None)
def createCommands(self):
## May add on help in children later on
self.commands = {
'get': ['content', 'props'],
'info': [],
'pull': ['file', 'directory'],
'put': ['file'],
'set': ['dhost', 'spaceID', 'user', 'validate', 'verbose', 'wsdl'],
'update': ['file'],
'validate': ['local', 'remote'],
'import': [],
'help': ['get', 'import', 'info', 'pull',
'put', 'set', 'update', 'validate']
}
## While we're here...
## Helper and Tab Complete objects
self.helper = Helper(self.commands)
self.completer = Completer(self.commands)
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind('tab: complete')
readline.set_completer(self.completer.complete)
def help(self, line):
self.helper.help(line)
def getCommands(self):
return self.commands
class Helper(object):
"""For all the help options that need to be printed"""
def __init__(self, commands):
self.commands = commands
def print_unknown(self, cmd, children, bad):
print 'Unknown option %s for %s' % (bad, cmd)
print 'Possible help command:'
for c in children:
print '\thelp %s %s' % (cmd, c)
def help_get(self, args):
children = self.commands.get('get')
if args[0] not in children:
self.print_unknown('get', children, args)
return
print 'get %s <file> [contentID]' % args[0]
if args[0] == 'content':
print '\tPrint the content of the file,',
print 'optionally specified by the contentID'
elif args[0] == 'props':
print '\tPrint the properties of the file,',
print 'optionally specified by the contentID'
def help_import(self, args):
if args:
self.print_unknown('import', [''], args)
return
print """import [tokenstore]
Import the token store to matching duraspace content
"""
def help_info(self, args):
if args:
self.print_unknown('info', [''], args)
return
print """info [param]
Print the shell information matching optional parameter"""
def help_pull(self, args):
children = self.commands.get('pull')
if not args or args[0] not in children:
self.print_unknown('pull', children, args)
return
print 'pull %s <file> [contentID]' % args[0]
if args[0] == 'file':
print '\tPull a file from durastore, optionally specified by contentID'
elif args[0] == 'directory':
print '\tPull a directory from durastore, optionally specified by contentID'
def help_put(self, args):
children = self.commands.get('put')
if not args or args[0] not in children:
self.print_unknown('put', children, args)
return
print 'put %s <file>' % args[0]
print '\tUpload a file or directory to durastore'
def help_set(self, args):
children = self.commands.get('set')
if not args or args[0] not in children:
self.print_unknown('set', children, args)
return
## There's probably a better way to do this... for now this works
if args[0] == 'wsdl':
print 'set %s [hostname]' % args[0]
print '\tSet the wsdl host, with optional hostname'
elif args[0] == 'dhost':
print 'set %s [hostname]' % args[0]
print '\tSet the durastore host, with optional hostname'
elif args[0] == 'spaceID':
print 'set %s [id]' % args[0]
print '\tSet the durastore space ID, with optional ID'
elif args[0] == 'user':
print 'set %s [user]' % args[0]
print '\tSet the durastore user, with optional username'
elif args[0] == 'validate':
print 'set %s' % args[0]
print '\tSet validation of files when pulling on or off'
elif args[0] == 'verbose':
print 'set %s' % args[0]
print '\tSet verbose mode on or off'
def help_update(self, args):
children = self.commands.get('update')
if not args or args[0] not in children:
self.print_unknown('update', children, args)
return
print 'update %s' % args[0]
print '\tUpdate the proofs for the file or directory file',
def help_validate(self, args):
children = self.commands.get('validate')
if not args or args[0] not in children:
self.print_unknown('validate', children, args)
return
if args[0] == 'local':
print 'validate %s <file> [contentID]' % args[0]
print '\tValidate a local file against it\'s proof in durastore, optional',
print 'contentID'
elif args[0] == 'remote':
print 'validate %s <contentID>' % args[0]
print '\tValidate the remote copy held in durastore specified by the',
print 'contentID'
def help_all(self, cmd):
print """Unknown command: """,cmd,"""
Possible help commands:
exit
help info
help set
help get
help put
help update
help validate
"""
def help(self, text):
line = list(text)
if not line:
self.help_all('')
return
cmd = line[0].strip()
if cmd in self.commands:
impl = getattr(self, 'help_%s' % cmd)
children = line[1:]
impl(children)
return
self.help_all(cmd)
## TODO: Bug when there is a space after a sub command, will match and add to
## the end of the line
class Completer(object):
def __init__(self, commands):
self.commands = commands
def complete_update(self, args):
cmds = self.commands.get('update')
## If a whole command is matched, return nozing
return [c + ' ' for c in cmds if not args or
(c.startswith(args[0]) and not args[0] in cmds)]
def complete_put(self, args):
cmds = self.commands.get('put')
return [c + ' ' for c in cmds if not args or
(c.startswith(args[0]) and not args[0] in cmds)]
def complete_set(self, args):
cmds = self.commands.get('set')
return [c + ' ' for c in cmds if not args or
(c.startswith(args[0]) and not args[0] in cmds)]
def complete_get(self, args):
cmds = self.commands.get('get')
return [c + ' ' for c in cmds if not args or
(c.startswith(args[0]) and not args[0] in cmds)]
def complete_validate(self, args):
cmds = self.commands.get('validate')
return [c + ' ' for c in cmds if not args or
(c.startswith(args[0]) and not args[0] in cmds)]
def complete_pull(self, args):
cmds = self.commands.get('pull')
return [c + ' ' for c in cmds if not args or
(c.startswith(args[0]) and not args[0] in cmds)]
def complete_help(self, args):
cmds = self.commands.get('help')
if not args:
return [c + ' ' for c in cmds]
subcmd = args[0].strip()
if subcmd not in cmds:
return [c + ' ' for c in cmds if c.startswith(subcmd)]
sargs = args[1:]
impl = getattr(self, 'complete_%s' % subcmd)
return impl(sargs)+[None]
def complete(self, text, state):
buf = readline.get_line_buffer()
line = readline.get_line_buffer().split()
## Really need to centralize these
# COMMANDS = ['help', 'exit', 'set', 'put', 'update', 'get', 'validate',
# 'pull']
COMMANDS = self.commands.keys()
if not line:
return [c + ' ' for c in COMMANDS][state]
regex = re.compile('.*\s+$', re.M)
if regex.match(buf):
line.append('')
cmd = line[0].strip()
if cmd in COMMANDS:
impl = getattr(self, 'complete_%s' % cmd)
args = filter(lambda x: x != '',line[1:])
return (impl(args)+[None])[state]
results = [c + ' ' for c in COMMANDS if c.startswith(cmd)] + [None]
return results[state]
#!/usr/bin/python2
#
# Shell to communication acetokens with duraspace and do some neato
# validation
#
import requests
import getopt, getpass, sys, os
import acestore, binascii
import acestore, acelib, binascii
import readline
import re
import xml.dom.minidom
from suds.client import Client
## TODO:
## Batch token creation on folders
## Help for individual commands
## Status Code to Human Readable
class AceShell:
"""Ace Shell Information"""
def __init__(self, user=None, passwd=None, dhost=None, space=None,
wsdl='http://ims.umiacs.umd.edu:8080/ace-ims/IMSWebService?wsdl',
verbose=False):
self.user = user
self.passwd = passwd
self.wsdl = wsdl
self.dhost = dhost
self.spaceID = space
self.verbose = verbose
self.run = True
self.validate = True
def printInfo(self):
print 'WSDL: ', self.wsdl, '\n',
print 'Duracloud Host: ', self.dhost, '\n',
print 'Duracloud SpaceID: ', self.spaceID, '\n',
print 'Duracloud Username: ', self.user, '\n',
print 'Validate on pull: ', self.validate, '\n',
print 'Verbose mode: ', self.verbose
def ready(self):
return (self.dhost!= None and self.spaceID != None and
self.user != None and self.passwd != None)
def shellHelp(shell):
print """Shell Commands:
help Show available commands
exit Quit the shell
info Show the shells parameters
set Set a parameter
wsdl [host] Set the wsdl host
dhost [host] Set the duracloud host
spaceID [id] Set the duracloud space
user [user] Set the duracloud user and password to
connect with
validate [file] Set validation of files when pulling from
duracloud
verbose Turn on/off verbose output
get
content [contentID] Get the content of the space, or
content if specified
props [contentID] Get the parameters of the space, or
content if specified
put
file [contentID] Upload a file to the duracloud host
directory Upload the contents of a directory to
the duracloud host
update
file [contendID] The file to update, and the contentID (optional)
validate
local [filename] Validate a local file
remote [contentID] Validate the remote file with the specified
contentID
"""
## Tab complete class
## Global for success codes
SUCCESS = (200, 201)
class Completer(object):
def complete_update(self, args):
cmds = ['file']
return [c + ' ' for c in cmds if not args or c.startswith(args[0])]
def complete_put(self, args):
cmds = ['file', 'directory']
return [c + ' ' for c in cmds if not args or c.startswith(args[0])]
def complete_set(self, args):
cmds = ['wsdl', 'dhost', 'spaceID', 'user', 'verbose']
return [c + ' ' for c in cmds if not args or c.startswith(args[0])]
def complete_get(self, args):
cmds = ['content', 'props']
return [c + ' ' for c in cmds if not args or c.startswith(args[0])]
def complete_validate(self, args):
cmds = ['remote', 'local']
return [c + ' ' for c in cmds if not args or c.startswith(args[0])]
def complete_pull(self, args):
cmds = ['file', 'directory']
return [c + ' ' for c in cmds if not args or c.startswith(args[0])]
def complete(self, text, state):
buf = readline.get_line_buffer()
line = readline.get_line_buffer().split()
COMMANDS = ['help', 'exit', 'set', 'put', 'update', 'get', 'validate',
'pull']
if not line:
return [c + ' ' for c in COMMANDS][state]
regex = re.compile('.*\s+$', re.M)
if regex.match(buf):
line.append('')
cmd = line[0].strip()
if cmd in COMMANDS:
impl = getattr(self, 'complete_%s' % cmd)
args = line[1:]
# Currently no commands have more than 1 argument, so chop it here
# Also, don't feel like completing path names hah
if len(args) > 1:
return
return (impl(args)+[None])[state]
results = [c + ' ' for c in COMMANDS if c.startswith(cmd)] + [None]
return results[state]
##
## Main functions for the shell commands
## On all - contentID should default to filename unless specified
## Will probably want to clean up how work on dir/files are done
## TODO: Unify style so it's not all over the place
##
## Open question -- should we use the base directory as the filename?
## ie: filename = /fs/narahomes/shake/scripts/ base = /scripts/
## on durastore as scripts/etcetc
def doPut(shell, arg, filename, contentID=None):
if os.path.isdir(filename):
if filename.endswith('/'):
filename = filename.rstrip('/')
os.path.walk(filename, processDir, (filename, 'put', shell))
return
"""Put files or directories into duraspace"""
if not os.path.exists(filename):
print 'File/Directory %s does not exist' % filename
return
pre_l = 'x-dura-meta-local-'
pre_r = 'x-dura-meta-remote-'
## Request lists, proof dicts, and client info
rl_l = createRequestList(filename, 'SHA-256')
rl_r = createRequestList(filename, 'MD5')
proof_l, client_l = acestore.getProof(rl_l)
proof_r, client_r = acestore.getProof(rl_r)
for file in prooflocal:
headers = createHeaders(pre_l, proof_l.get(file), client_l)
headers.update(createHeaders(pre_r, proof_r.get(file), client_r))
headers = createHeaders(filename)
if contentID == None:
contentID = filename
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID+'/'+contentID
req = requests.put(url, auth=(shell.user, shell.passwd),
headers=headers, data=open(filename, 'rb'))
print req.status_code
contentID = file
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID+'/'+contentID
req = requests.put(url, auth=(shell.user, shell.passwd),
headers=headers, data=open(file, 'rb'))
if req.status_code in SUCCESS:
print 'Successfully put %s' % file
else:
print 'Could not put %s into your durastore space' % file
## TODO: This and put share many things... combine
def doUpdate(shell, arg, filename, contentID=None):
if not os.path.isfile(filename) and not os.path.isdir(filename):
print "File/Directory",filename,"does not exist"
return
"""Update files or directories already in duraspace"""
if os.path.isdir(filename):
if filename.endswith('/'):
filename = filename.rstrip('/')
os.path.walk(filename, processDir, (filename, 'post', shell))
if not os.path.exists(filename):
print 'File/Directory %s does not exist' % filename
return
headers = createHeaders(filename)
if contentID == None:
contentID = filename
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID+'/'+contentID
print url
req = requests.post(url, auth=(shell.user, shell.passwd),
headers=headers)
print req.status_code
pre_l = 'x-dura-meta-local-'
pre_r = 'x-dura-meta-remote-'
## Request lists, proof dicts, and client info
rl_l = createRequestList(filename, 'SHA-256')
rl_r = createRequestList(filename, 'MD5')
proof_l, client_l = acestore.getProof(rl_l)
proof_r, client_r = acestore.getProof(rl_r)
for file in prooflocal:
headers = createHeaders(pre_l, proof_l.get(file), client_l)
headers.update(createHeaders(pre_r, proof_r.get(file), client_r))
contentID = file
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID+'/'+contentID
req = requests.post(url, auth=(shell.user, shell.passwd), headers=headers)
if req.status_code in SUCCESS:
print 'Successfully updated %s' % filename
else:
print 'Could not update %s' % filename
# Similar to pull, but we're not saving the file... not sure what the
# point of that is...
# point of that is, especially if the file is not text
# TODO: Pretty print for headers
def doGet(shell, type, contentID=None):
"""Print contents or properties of files in duraspace"""
req = None
url = "https://"+shell.dhost+"/durastore/"+shell.spaceID
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID
if contentID != None:
url += "/"+contentID
......@@ -181,13 +102,36 @@ def doGet(shell, type, contentID=None):
elif type == 'props':
req = requests.head(url, auth=(shell.user, shell.passwd))
print "Status: \n", req.status_code
status = 'success' if req.status_code in SUCCESS else 'failure'
print "Status: \n", status
if shell.verbose or type =='props':
print "Headers: \n", req.headers
print "Data: \n", req.text
def doImport(shell, filename):
if not os.path.exists(filename):
print 'Cannot open file %s' % filename
return
pre = 'x-dura-meta-local-'
base = raw_input('Enter the base contentID of the duraspace collection (leave blank for none): ')
ts = acestore.TokenStore(open(filename, 'r'))
for file in ts.entries:
entry = ts.entries.get(file)
contentID = os.path.join(base, file.lstrip('/'))
headers = createHeaders(pre, entry.proof, entry.get_client_info())
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID+'/'+contentID
req = requests.post(url, auth=(shell.user, shell.passwd), headers=headers)
if req.status_code in SUCCESS:
print 'Successfully added ace proof to %s' % file
else:
print 'Could not add ace proof for %s' % file
print 'Tried to use contentID: %s' % contentID
def doSet(shell, arg, param=None):
"""Set shell properties"""
if arg == 'wsdl':
shell.wsdl = (raw_input('WSDL Host: ')
if param == None else param)
......@@ -228,60 +172,84 @@ def doSet(shell, arg, param=None):
def doPull(shell, arg, file, contentID=None):
"""Pull files and directories from duraspace"""
debug = True
stage = raw_input('Where would you like to stage the data? ')
if arg == 'directory':
print 'I really shouldn\'t add these in until I\'ve coded them'
return
# Get the file name and make a request
# Not sure how this would fare for large files ( >8G or so )
if contentID == None:
contentID = file
file = os.path.join(stage, contentID)
print file
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID+'/'+contentID
req = requests.get(url, auth=(shell.user, shell.passwd))
if req.status_code == 404:
print "File Not Found in your duraspace."
return
## Let's grab content from the space
## Then match against everything in the dir and pull them
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID
req = requests.get(url, auth=(shell.user, shell.passwd))
dom = xml.dom.minidom.parseString(req.text)
items = dom.getElementsByTagName('item')
contentIDs = [item.firstChild.nodeValue for item in items if
item.firstChild.nodeValue.startswith(file)]
else:
contentIDs = [contentID]
for contentID in contentIDs:
## Get the file name and make a request
## Not sure how this would fare for large files ( >8G or so )
if contentID == None:
contentID = file
file = os.path.join(stage, contentID)
print file
url = 'https://'+shell.dhost