View of /offline/Production/panda/pilot2/dmu/LocalSiteMover.py

import os, re, sys
import commands

"""
Questions:
in get_data the file name is controlled by the invoker?
"""

import SiteMover
from futil import *
from PilotErrors import PilotErrors
from pUtil import tolog, readpar
from TimedCommand import timed_command

class localSiteMover(SiteMover.SiteMover):
    """ SiteMover for local copy commands etc """
    
    copyCommand = "lsm"
    __warningStr = '!!WARNING!!2995!! %s' # warning string
    __errorStr = '!!ERROR!!2995!! %s' # error string
    __spacetoken = '-t %s' # space token descriptor
    __localget = '%s lsm-get %s %s' # environment, lfn, target directory
    __localput = '%s lsm-put %s %s %s' # environment, space token (optional), source directory, destination
    __localspace = '%s lsm-df %s %s' # environment, space token (optional), storage end-point
    __localerror = 'lsm-error %d' # error code
    __par_filesize = ' --size %s' # filesize in bytes
    __par_checksum = ' --checksum %s' # checksum string: "adler32:NNN", "md5:NNN", default is assumed MD5
    __timeout = 1800 # seconds
    __error = PilotErrors()
    __pilotErrorDiag = ''

    def check_space(self, whatever):
        # build setup string
        envsetup = readpar('envsetup')
        tmp = envsetup.strip()
        if tmp != "" and not tmp.endswith(';'):
            tmp += ";"
        envsetup = tmp.replace(";;",";")

        # build the df command
        execStr = self.__localspace % (envsetup, '', whatever)

        # execute
        try:
            status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog(self.__warningStr % self.__pilotErrorDiag)            
            status = 1
            output = str(e)
            telapsed = self.__timeout
        else:
            # improve output parsing, keep stderr and stdout separate
            output = cout + cerr
        tolog("Elapsed time: %d" % (telapsed))

        # validate
        if status:
            self.__pilotErrorDiag = 'lsm-df failed (%s): %s' % (status, output)
            tolog(self.__errorStr % self.__pilotErrorDiag)            
            # use different error code
            return 999999
        try:
            return int(output.strip())
        except:
            self.__pilotErrorDiag = 'lsm-df wrong format (%s): %s' % (status, output)
            tolog(self.__errorStr % self.__pilotErrorDiag)            
            # use different error code
        return 999999

    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """ copy input file from SE to local dir """

        if not path:
            tolog('path is empty, using current directory')            
            path = os.getcwd()

        # build setup string
        envsetupin = readpar('envsetupin')
        if envsetupin == "":
            tolog("Using envsetup since envsetupin is not set")
            envsetup = readpar('envsetup')
        else:
            envsetup = envsetupin

        tmp = envsetup.strip()
        if tmp != "" and not tmp.endswith(';'):
            tmp += ";"
        if os.environ.has_key('X509_USER_PROXY'):
            tmp += "export X509_USER_PROXY=%s;" % (os.environ['X509_USER_PROXY'])
        envsetup = tmp.replace(";;",";")

        # build the get command
        execStr = self.__localget % (envsetup, gpfn, path)

        # execute
        try:
            status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog(self.__warningStr % self.__pilotErrorDiag)
            status = 1
            output = str(e)
            telapsed =self.__timeout
        else:
            # improve output parsing, keep stderr and stdout separate
            output = cout + cerr
        tolog("Elapsed time: %d" % (telapsed))

        # validate
        if status:
            self.__pilotErrorDiag = 'lsm-get failed (%s): %s' % (status, output)
            tolog(self.__errorStr % self.__pilotErrorDiag)            
            # use different error code
            # remove partial copy? is it possible?
            return self.__error.ERR_DQ2GETFAIL, self.__pilotErrorDiag

        # checksum and filesize check done by lsm-get

        # return gracefully

        return 0, self.__pilotErrorDiag
    
    def put_data(self, source, destination, fsize=0, fchecksum=0, dsname='', extradirs='', **pdict):
        """ copy output file from local dir to SE and register into dataset and catalogues """
        
        # get the space token description if available
        try:
            token = pdict['token']
        except:
            token = ""

        # get the local file size and checksum
        if fsize == 0 or fchecksum == 0:
            ec, pilotErrorDiag, fsize, fchecksum = SiteMover.getLocalFileInfo(source, csumtype=csumtype)
            if ec != 0:
                return SiteMover.put_data_retfail(ec, pilotErrorDiag)
        
        # get the checksum type
        if fchecksum != 0 and fchecksum != "":
            csumtype = SiteMover.getChecksumType(fchecksum)
        else:
            csumtype = "default"

        # setup string
        envsetup = readpar('envsetup')

        tmp = envsetup.strip()
        if tmp != "" and not tmp.endswith(';'):
            tmp += ";"
        if os.environ.has_key('X509_USER_PROXY'):
            tmp += "export X509_USER_PROXY=%s;" % (os.environ['X509_USER_PROXY'])
        envsetup = tmp.replace(";;",";")


        # build the command
        _params = ""
        if token:
            _params = self.__spacetoken % (token,)
        if fsize != 0 :
            _params += self.__par_filesize % (fsize,)
        if fchecksum != 0 :
            if csumtype == "default":
                _params += self.__par_checksum % (fchecksum,)
            else:
                _params += self.__par_checksum % ("%s:%s" % (csumtype, fchecksum),)
        execStr = self.__localput % (envsetup, _params, source, destination)
        
        # execute
        try:
            status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog(self.__warningStr % self.__pilotErrorDiag)
            status = 1
            output = str(e)
            telapsed = self.__timeout
        else:
            # manage better output? leave stdout and stderr separate?
            output = cout + cerr
        tolog("Elapsed time: %d" % (telapsed))

        # validate
        
        if status:
            self.__pilotErrorDiag = 'lsm-put failed (%s): %s' % (status, output)
            tolog(self.__errorStr % self.__pilotErrorDiag)
            # remove the file with lsm-rm? is partial copy possible?
            return SiteMover.put_data_retfail(__error.ERR_FAILEDCP, self.__pilotErrorDiag)

        # checksum and filesize check done by lsm-put

        #return gracefully

        return 0, self.__pilotErrorDiag, destination, fsize, fchecksum, ARCH_DEFAULT