Commit 9674208f authored by shake's avatar shake
Browse files

Added ability to pull from durastore to local file

git-svn-id: https://subversion.umiacs.umd.edu/ace/trunk@192 f1b3a171-7291-4a19-a512-95ad0ad9394a
parent 3b1edac8
#!/fs/narahomes/shake/built/bin/python
#!/usr/bin/python2
import requests
import getopt, getpass, sys, os
......@@ -9,6 +9,7 @@ from suds.client import Client
class AceShell:
"""Ace Shell Information"""
def __init__(self, user=None, passwd=None, wsdl=None, dhost=None,
space=None, verbose=False):
self.user = user
......@@ -24,7 +25,9 @@ class AceShell:
print 'WSDL: ', self.wsdl, '\n',
print 'Duracloud Host: ', self.dhost, '\n',
print 'Duracloud SpaceID: ', self.spaceID, '\n',
print 'Duracloud Username: ', self.user
print 'Duracloud Username: ', self.user, '\n',
print 'Validate on pull: ', self.validate, '\n',
print 'Verbose mode: ', self.verbose
def ready(self):
return (self.dhost!= None and self.spaceID != None and
......@@ -35,12 +38,22 @@ def shellHelp(shell):
help Show available commands
exit Quit the shell
info Show the shells parameters
set Set a parameter
wsdl [host] The wsdl host
dhost [host] The duracloud host
spaceID [id] The duracloud space
user [user] The duracloud user and password to
connect with
validate Validate files when pulling from
duracloud
verbose Verbose output
get
content [contentID] Get the content of the space, or
content if specified
props [contentID] Get the parameters of the space, or
content if specified
put
file [contentID] Upload a file to the duracloud host
directory Upload the contents of a directory to
......@@ -48,11 +61,6 @@ def shellHelp(shell):
update
file [contendID] The file to update, and the contentID (optional)
get
content [contentID] Get the content of the space, or
content if specified
props [contentID] Get the parameters of the space, or
content if specified
validate
local [filename] Validate a local file
remote [contentID] Validate the remote file with the specified
......@@ -60,6 +68,8 @@ def shellHelp(shell):
"""
class Completer(object):
"""Completer for tab complete info"""
def complete_update(self, args):
cmds = ['file']
return [c + ' ' for c in cmds if not args or c.startswith(args[0])]
......@@ -80,11 +90,16 @@ class Completer(object):
cmds = ['remote', 'local']
return [c + ' ' for c in cmds if not args or c.startswith(args[0])]
def complete_pull(self, args):
cmds = ['file', 'directory']
return [c + ' ' for c in cmds if not args or c.startswith(args[0])]
def complete(self, text, state):
buf = readline.get_line_buffer()
line = readline.get_line_buffer().split()
COMMANDS = ['help', 'exit', 'set', 'put', 'update', 'get', 'validate']
COMMANDS = ['help', 'exit', 'set', 'put', 'update', 'get', 'validate',
'pull']
if not line:
return [c + ' ' for c in COMMANDS][state]
......@@ -103,38 +118,9 @@ class Completer(object):
results = [c + ' ' for c in COMMANDS if c.startswith(cmd)] + [None]
return results[state]
def createHeaders(filename):
pre = 'x-dura-meta-'
headers = {}
reqList = [(filename, binascii.b2a_hex(acestore.digestFile(filename)))]
prooflist, clientinfo = acestore.getProof(reqList)
headers[pre+'roundId'] = clientinfo['roundId']
headers[pre+'tokenClass'] = clientinfo['tokenClass']
headers[pre+'digestService'] = clientinfo['digestService']
proof = prooflist[filename]
for line in proof:
headers[pre+"proof-"+str(proof.index(line))] = line
return headers
def processDir(args, directory, files):
origin, method, shell = args
req = None
for f in files:
_, base = os.path.split(origin)
_, _, dirs = directory.partition(base)
contentID = base+dirs+'/'+f
if (os.path.isdir(os.path.join(directory, f))):
continue
headers = createHeaders(os.path.join(directory,f))
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID+'/'+contentID
if method == 'post':
req = requests.post(url, auth=(shell.user, shell.passwd),
headers=headers)
elif method == 'put':
fobj = open(os.path.join(directory, f), 'rb')
req = requests.put(url, auth=(shell.user, shell.passwd),
data=fobj, headers=headers)
print req.url, req.status_code,'\n', req.text
##
## Main functions for the shell commands
##
def doPut(shell, arg, filename, contentID=None):
if not shell.ready():
......@@ -155,8 +141,7 @@ def doPut(shell, arg, filename, contentID=None):
headers=headers, data=open(filename, 'rb'))
print req.status_code
def update(shell, arg, filename, contentID=None):
def doUpdate(shell, arg, filename, contentID=None):
if not shell.ready():
print "Cannot perform update without proper information set first"
return
......@@ -166,7 +151,6 @@ def update(shell, arg, filename, contentID=None):
return
if os.path.isdir(filename):
print "Updating directory recursively is not yet supported"
if filename.endswith('/'):
filename = filename.rstrip('/')
os.path.walk(filename, processDir, (filename, 'post', shell))
......@@ -179,6 +163,8 @@ def update(shell, arg, filename, contentID=None):
req = requests.post(url, auth=(shell.user, shell.passwd),
headers=headers)
# Similar to pull, but we're not saving the file... not sure what the
# point of that is...
def doGet(shell, type, contentID=None):
if not shell.ready():
print "Cannot perform get without proper information set first"
......@@ -195,7 +181,8 @@ def doGet(shell, type, contentID=None):
req = requests.head(url, auth=(shell.user, shell.passwd))
print "Status: \n", req.status_code
print "Headers: \n", req.headers
if verbose or type =='props':
print "Headers: \n", req.headers
print "Data: \n", req.text
def doSet(shell, arg, param=None):
......@@ -223,13 +210,42 @@ def doSet(shell, arg, param=None):
elif arg == 'verbose':
if shell.verbose:
print 'Turning off verbose mode.'
shell.verbose = False
else:
print 'Turning on verbose mode.'
shell.verbose = True
shell.verbose = not shell.verbose
elif arg == 'validate':
if shell.validate:
print 'Turning off validate on pull.'
else:
print 'Turning on validate on pull.'
shell.validate = not shell.validate
else:
print "Unknown set parameter."
def doPull(shell, arg, file):
print "Pulling"
stage = raw_input('Where would you like to stage the data? ')
if arg == 'directory':
print 'I really shouldn\'t add these in until I\'ve coded them'
return
# Get the file name and make a request
# Not sure how this would fare for large files ( >8G or so )
_, contentID = os.path.split(file)
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID+'/'+contentID
req = requests.get(url, auth=(shell.user, shell.passwd))
if req.status_code == 404:
print "File Not Found in your duraspace."
return
fnew = open(stage+'/'+contentID, 'w+b')
fnew.write(req.content)
fnew.close()
print req.status_code, 'finished pulling data'
def doValidate(shell, arg, file):
pre = 'x-dura-meta-'
if os.path.isdir(file):
......@@ -240,11 +256,13 @@ def doValidate(shell, arg, file):
print 'Validating remote files will be supported soon'
return
url = "https://"+shell.dhost+"/durastore/"+shell.spaceID+"/"+file
_, contentID = os.path.split(file)
url = "https://"+shell.dhost+"/durastore/"+shell.spaceID+"/"+contentID
req = requests.head(url, auth=(shell.user, shell.passwd))
headers = req.headers
keys = headers.keys()
keys.sort()
proof = [headers.get(x) for x in keys
if x.startswith(pre+'proof')]
digest = headers.get(pre+'digestservice')
......@@ -259,6 +277,22 @@ def doValidate(shell, arg, file):
print rhash == csi
##
## Helper methods for the most part
##
def checkCSI(roundid, proof, digest, file):
""" To be used for validating eventually"""
csi = acestore.calculateCSI(file, proof, digest)
roundhash = getRoudnHash(roundid)
if shell.verbose:
print 'Round ID:', roundid
print 'Round Hash from IMS:', roundhash
print 'Calculated csi:',csi
print file, ' is valid:', rhash == csi
def getRoundHash(round, url=None):
url = 'http://ims.umiacs.umd.edu:8080/ace-ims/IMSWebService?wsdl'
......@@ -266,6 +300,41 @@ def getRoundHash(round, url=None):
response = client.service.getRoundSummaries(round)
return response.__getitem__(0).hashValue
# Need to check working on a directory when given a full path
def processDir(args, directory, files):
origin, method, shell = args
req = None
for f in files:
_, base = os.path.split(origin)
_, _, dirs = directory.partition(base)
contentID = base+dirs+'/'+f
# Ignore directories...
if (os.path.isdir(os.path.join(directory, f))):
continue
headers = createHeaders(os.path.join(directory,f))
url = 'https://'+shell.dhost+'/durastore/'+shell.spaceID+'/'+contentID
if method == 'post':
req = requests.post(url, auth=(shell.user, shell.passwd),
headers=headers)
elif method == 'put':
fobj = open(os.path.join(directory, f), 'rb')
req = requests.put(url, auth=(shell.user, shell.passwd),
data=fobj, headers=headers)
print req.url, req.status_code,'\n', req.text
def createHeaders(filename):
pre = 'x-dura-meta-'
headers = {}
reqList = [(filename, binascii.b2a_hex(acestore.digestFile(filename)))]
prooflist, clientinfo = acestore.getProof(reqList)
headers[pre+'roundId'] = clientinfo['roundId']
headers[pre+'tokenClass'] = clientinfo['tokenClass']
headers[pre+'digestService'] = clientinfo['digestService']
proof = prooflist[filename]
for line in proof:
headers[pre+"proof-"+str(proof.index(line))] = line
return headers
def printInfo(shell):
shell.printInfo()
......@@ -288,17 +357,25 @@ def process(cmd, shell):
if cmd == None or len(cmd) == 0: return
fns = {'help': shellHelp, 'exit': stopShell,
'info': printInfo, 'update': update,
'info': printInfo, 'update': doUpdate,
'get': doGet, 'set': doSet,
'put': doPut, 'validate': doValidate}
'put': doPut, 'pull': doPull,
'validate': doValidate}
args = cmd.split()
if args[0] not in fns:
print args[0], "is not a valid command"
return
if args[0] in ['get', 'update', 'set', 'put', 'validate', 'pull'] and not shell.ready():
print "Please set duraspace information before using",args[0]
return
func = fns[args[0]]
func(shell, *args[1:])
# For running while coding
##
## For running while coding
##
# def repl():
# global acestore
# acestore = reload(acestore)
......@@ -317,13 +394,19 @@ def main():
shell = AceShell()
prompt = '>>\033[1;37m '
# Set up our tab complete object
comp = Completer()
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind('tab: complete')
readline.set_completer(comp.complete)
print "Ace Shell"
print "To get a list of commands type help, to leave type exit"
# This gets a little buggy, should fix when the input gets fubar'd
# Possibly from the completer adding in things to the prompt?
while shell.run:
sys.stdout.write(prompt),
cmd = raw_input(prompt) # sys.stdin.readline().strip('\n') # raw_input()
cmd = raw_input(prompt)
sys.stdout.write('\033[0m')
process(cmd, shell)
......
#!/usr/local/stow/python-2.7.3/bin/python
#!/usr/bin/python2
#
# ACE Token store tool
#
......@@ -127,7 +127,7 @@ def getAlgorithm(algName):
def calculateCSI(file, proof, algName):
"""Calculate the CSI for a file"""
prevhash = digestFile(file)
prevhash = digestFile(file, algName)
for proofLine in proof:
prevhash = calculateLevel(prevhash,proofLine,algName)
return binascii.b2a_hex(prevhash)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment