Goffi's cp, a fancy file copier
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

697 lines
27 KiB

#!/usr/bin/env python3
11 years ago
"""
gcp: Goffi's CoPier
Copyright (C) 2010, 2011 Jérôme Poisson <goffi@goffi.org>
(c) 2011 Thomas Preud'homme <robotux@celest.fr>
11 years ago
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
### logging ###
import logging
from logging import debug, info, error, warning
logging.basicConfig(level=logging.INFO,
11 years ago
format='%(message)s')
###
import gettext
gettext.install('gcp', "i18n")
11 years ago
import sys
import os,os.path
from argparse import ArgumentParser
import pickle
11 years ago
try:
from gi.repository import GObject
11 years ago
#DBus
import dbus, dbus.glib
import dbus.service
import dbus.mainloop.glib
except ImportError as e:
error(_("Error during import"))
error(_("Please check dependecies:"),e)
exit(1)
11 years ago
try:
from progressbar import ProgressBar, Percentage, Bar, ETA, FileTransferSpeed
pbar_available=True
except ImportError as e:
info (_('ProgressBar not available, please download it at http://pypi.python.org/pypi/progressbar'))
info (_('Progress bar deactivated\n--\n'))
11 years ago
pbar_available=False
NAME = "gcp (Goffi's copier)"
NAME_SHORT = "gcp"
VERSION = '0.1.3'
11 years ago
ABOUT = NAME+u" v"+VERSION+u""" (c) Jérôme Poisson (aka Goffi) 2010, 2011
11 years ago
---
"""+NAME+u""" Copyright (C) 2010 Jérôme Poisson
""" + _(u"""This program comes with ABSOLUTELY NO WARRANTY;
11 years ago
This is free software, and you are welcome to redistribute it
under certain conditions.
---
This software is an advanced file copier
Get the latest version at http://wiki.goffi.org/wiki/Gcp
""")
11 years ago
const_DBUS_INTERFACE = "org.goffi.gcp"
const_DBUS_PATH = "/org/goffi/gcp"
const_BUFF_SIZE = 4096
const_PRESERVE = set(['mode','ownership','timestamps'])
const_FS_FIX = set(['auto','force','no'])
const_FILES_DIR = "~/.gcp"
const_JOURNAL_PATH = const_FILES_DIR + "/journal"
const_SAVED_LIST = const_FILES_DIR + "/saved_list"
11 years ago
11 years ago
class DbusObject(dbus.service.Object):
def __init__(self, gcp, bus, path):
self._gcp = gcp
dbus.service.Object.__init__(self, bus, path)
debug(_("Init DbusObject..."))
11 years ago
self.cb={}
11 years ago
@dbus.service.method(const_DBUS_INTERFACE,
in_signature='', out_signature='s')
def getVersion(self):
"""Get gcp version
@return: version as string"""
return VERSION
@dbus.service.method(const_DBUS_INTERFACE,
in_signature='ss', out_signature='bs')
def addArgs(self, source_dir, args):
11 years ago
"""Add arguments to gcp as if there were entered on its own command line
@param source_dir: current working dir to use as base for arguments, as given by os.getcwd()
@param args: serialized (wich pickle) list of strings - without command name -, as given by sys.argv[1:].
@return: success (boolean) and error message if any (string)"""
try:
args = pickle.loads(str(args))
except TypeError as e:
pickle.UnpicklingError = e
return (False, _("INTERNAL ERROR: invalid arguments"))
try:
source_dir = pickle.loads(str(source_dir))
except TypeError as e:
pickle.UnpicklingError = e
return (False, _("INTERNAL ERROR: invalid source_dir"))
return self._gcp.parseArguments(args, source_dir)
11 years ago
class Journal():
def __init__(self, path=const_JOURNAL_PATH):
self.journal_path = os.path.expanduser(path)
self.journal_fd = open(self.journal_path,'w') #TODO: check and maybe save previous journals
self.__entry_open = None
self.failed = []
self.partial = []
def __del__(self):
self.journal_fd.flush()
self.journal_fd.close()
def startFile(self, source_path):
"""Start an entry in the journal"""
assert not self.__entry_open
self.__entry_open = source_path
self.journal_fd.write(source_path+"\n")
self.journal_fd.flush()
self.success=True
self.errors=[]
def closeFile(self):
"""Close the entry in the journal"""
assert self.__entry_open
if not self.success:
status = "FAILED"
else:
status = "OK" if not self.errors else "PARTIAL"
self.journal_fd.write("%(status)s: %(errors)s\n" % {'status': status, 'errors': ', '.join(self.errors)})
self.journal_fd.flush()
self.__entry_open = None
def copyFailed(self):
"""Must be called when something is wrong with the copy itself"""
assert self.__entry_open
self.success = False
self.failed.append(self.__entry_open)
def error(self, name):
"""Something went wrong"""
assert self.__entry_open
self.errors.append(name)
self.partial.append(self.__entry_open)
def showErrors(self):
"""Show which files were not successfully copied"""
failed = set(self.failed)
partial = set(self.partial)
for entry in failed:
partial.discard(entry)
if failed:
error(_("/!\\ THE FOLLOWING FILES WERE *NOT* SUCCESSFULY COPIED:"))
#TODO: use logging capability to print all error message in red
for entry in failed:
info("\t- %s" % entry)
info ('--\n')
if partial:
warning(_("The following files were copied, but some errors happened:"))
for entry in partial:
info("\t- %s" % entry)
info ('--\n')
if failed or partial:
info(_("Please check journal: %s") % self.journal_path)
11 years ago
class GCP():
def __init__(self):
files_dir = os.path.expanduser(const_FILES_DIR)
if not os.path.exists(files_dir):
os.makedirs(files_dir)
11 years ago
try:
sessions_bus = dbus.SessionBus()
db_object = sessions_bus.get_object(const_DBUS_INTERFACE,
const_DBUS_PATH)
self.gcp_main = dbus.Interface(db_object,
dbus_interface=const_DBUS_INTERFACE)
self._main_instance = False
except dbus.exceptions.DBusException as e:
11 years ago
if e._dbus_error_name=='org.freedesktop.DBus.Error.ServiceUnknown':
self.launchDbusMainInstance()
debug (_("gcp launched"))
11 years ago
self._main_instance = True
self.buffer_size = const_BUFF_SIZE
self.__launched = False #True when journal is initialised and copy is started
11 years ago
else:
raise e
def launchDbusMainInstance(self):
debug (_("Init DBus..."))
11 years ago
session_bus = dbus.SessionBus()
self.dbus_name = dbus.service.BusName(const_DBUS_INTERFACE, session_bus)
self.dbus_object = DbusObject(self, session_bus, const_DBUS_PATH)
self.copy_list = []
self.mounts = self.__getMountPoints()
self.bytes_total = 0
self.bytes_copied = 0
11 years ago
def getFsType(self, path):
fs = ''
last_mount_point = ''
11 years ago
for mount in self.mounts:
if path.startswith(mount) and len(mount)>=len(last_mount_point):
11 years ago
fs = self.mounts[mount]
last_mount_point = mount
11 years ago
return fs
def __getMountPoints(self):
"""Parse /proc/mounts to get currently mounted devices"""
#TODO: reparse when a new device is added/a device is removed
#(check freedesktop mounting signals)
11 years ago
ret = {}
try:
with open("/proc/mounts",'r') as mounts:
11 years ago
for line in mounts.readlines():
fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno = line.split(' ')
ret[fs_file] = fs_vfstype
except:
error (_("Can't read mounts table"))
11 years ago
return ret
def __appendToList(self, path, dest_path, options):
11 years ago
"""Add a file to the copy list
@param path: absolute path of file
@param options: options as return by optparse"""
debug (_("Adding to copy list: %(path)s ==> %(dest_path)s (%(fs_type)s)") % {"path":path, "dest_path":dest_path, "fs_type":self.getFsType(dest_path)} )
try:
self.bytes_total+=os.path.getsize(path)
self.copy_list.insert(0,(path, dest_path, options))
except OSError as e:
error(_("Can't copy %(path)s: %(exception)s") % {'path':path, 'exception':e.strerror})
11 years ago
def __appendDirToList(self, dirpath, dest_path, options):
11 years ago
"""Add recursively directory to the copy list
@param path: absolute path of dir
@param options: options as return by optparse"""
#We first check that the dest path exists, and create it if needed
dest_path = self.__filename_fix(dest_path, options, no_journal=True)
if not os.path.exists(dest_path):
debug ("Creating directory %s" % dest_path)
os.makedirs(dest_path) #TODO: check permissions
#TODO: check that dest_path is an accessible dir,
# and skip file/write error in log if needed
11 years ago
try:
for filename in os.listdir(dirpath):
filepath = os.path.join(dirpath,filename)
if os.path.islink(filepath) and not options.dereference:
debug ("Skippink symbolic dir: %s" % filepath)
continue
11 years ago
if os.path.isdir(filepath):
full_dest_path = os.path.join(dest_path,filename)
self.__appendDirToList(filepath, full_dest_path, options)
11 years ago
else:
self.__appendToList(filepath, dest_path, options)
except OSError as e:
try:
error(_("Can't append %(path)s to copy list: %(exception)s") % {'path':filepath, 'exception':e.strerror})
except NameError:
#We can't list the dir
error(_("Can't access %(dirpath)s: %(exception)s") % {'dirpath':dirpath, 'exception':e.strerror})
11 years ago
def __checkArgs(self, options, source_dir, args):
"""Check thats args are files, and add them to copy list
@param options: options sets
@param source_dir: directory where the command was entered
@parm args: args of the copy"""
assert(len (args)>=2)
len_args = len(args)
try:
dest_path = os.path.normpath(os.path.join(source_dir, args.pop()))
except OSError as e:
error (_("Invalid dest_path: %s"),e)
11 years ago
for path in args:
abspath = os.path.normpath(os.path.join(os.path.expanduser(source_dir), path))
11 years ago
if not os.path.exists(abspath):
warning(_("The path given in arg doesn't exist or is not accessible: %s") % abspath)
11 years ago
else:
if os.path.isdir(abspath):
if not options.recursive:
warning (_('omitting directory "%s"') % abspath)
11 years ago
else:
_basename=os.path.basename(os.path.normpath(path))
full_dest_path = dest_path if options.directdir else os.path.normpath(os.path.join(dest_path, _basename))
self.__appendDirToList(abspath, full_dest_path, options)
11 years ago
else:
self.__appendToList(abspath, dest_path, options)
def __copyNextFile(self):
"""Take the last file in the list, and launch the copy using glib io_watch event
@return: True a file was added, False else"""
if self.copy_list:
source_file, dest_path, options = self.copy_list.pop()
self.journal.startFile(source_file)
try:
source_fd = open(source_file, 'rb')
except:
self.journal.copyFailed()
self.journal.error("can't open source")
self.journal.closeFile()
return True
filename = os.path.basename(source_file)
assert(filename)
dest_file = self.__filename_fix(options.dest_file,options) if options.dest_file else self.__filename_fix(os.path.join(dest_path,filename),options)
if os.path.exists(dest_file) and not options.force:
warning (_("File [%s] already exists, skipping it !") % dest_file)
self.journal.copyFailed()
self.journal.error("already exists")
self.journal.closeFile()
source_fd.close()
return True
try:
dest_fd = open(dest_file, 'wb')
except:
self.journal.copyFailed()
self.journal.error("can't open dest")
self.journal.closeFile()
source_fd.close()
return True
GObject.io_add_watch(source_fd, GObject.IO_IN,self._copyFile,
(dest_fd, options), priority=GObject.PRIORITY_DEFAULT)
if not self.progress:
info(_("COPYING %(source)s ==> %(dest)s") % {"source":source_file, "dest":dest_file})
return True
else:
#Nothing left to copy, we quit
if self.progress:
self.__pbar_finish()
self.journal.showErrors()
self.loop.quit()
def __copyFailed(self, reason, source_fd, dest_fd):
"""Write the failure in the journal and close files descriptors"""
self.journal.copyFailed()
self.journal.error(reason)
self.journal.closeFile()
source_fd.close()
dest_fd.close()
def _copyFile(self, source_fd, condition, data):
"""Actually copy the file, callback used with io_add_watch
@param source_fd: file descriptor of the file to copy
@param condition: condition which launched the callback (glib.IO_IN)
@param data: tuple with (destination file descriptor, copying options)"""
try:
dest_fd,options = data
try:
buff = source_fd.read(self.buffer_size)
except KeyboardInterrupt:
raise KeyboardInterrupt
except:
self.__copyFailed("can't read source", source_fd, dest_fd)
return False
try:
dest_fd.write(buff)
except KeyboardInterrupt:
raise KeyboardInterrupt
except:
self.__copyFailed("can't write to dest", source_fd, dest_fd)
return False
self.bytes_copied += len(buff)
if self.progress:
self._pbar_update()
if len(buff) != self.buffer_size:
source_fd.close()
dest_fd.close()
self.__post_copy(source_fd.name, dest_fd.name, options)
self.journal.closeFile()
return False
return True
except KeyboardInterrupt:
self._userInterruption()
11 years ago
def __filename_fix(self, filename, options, no_journal=False):
"""Fix filenames incompatibilities/mistake according to options
@param filename: full path to the file
@param options: options as parsed on command line
@param no_journal: don't write any entry in journal
@return: fixed filename"""
fixed_filename = filename
if options.fs_fix == 'force' or (options.fs_fix == 'auto' and self.getFsType(filename) == 'vfat'):
fixed_filename = filename.replace('\\','_')\
.replace(':',';')\
.replace('*','+')\
.replace('?','_')\
.replace('"','\'')\
.replace('<','[')\
.replace('>',']')\
.replace('|','!')\
.rstrip() #XXX: suffixed spaces cause issues (must check FAT doc for why)
if not fixed_filename:
fixed_filename = '_'
if fixed_filename != filename and not no_journal:
self.journal.error('filename fixed')
return fixed_filename
def __post_copy(self, source_file, dest_file, options):
"""Do post copy traitement (mainly managing --preserve option)"""
st_file = os.stat(source_file)
for preserve in options.preserve:
try:
if preserve == 'mode':
os.chmod(dest_file, st_file.st_mode)
elif preserve == 'ownership':
os.chown(dest_file, st_file.st_uid, st_file.st_gid)
elif preserve == 'timestamps':
os.utime(dest_file, (st_file.st_atime, st_file.st_mtime))
except OSError as e:
self.journal.error("preserve-"+preserve)
def __get_string_size(self, size):
"""Return a nice string representation of a size"""
if size>=2**50:
return _("%.2f PiB") % (float(size)/2**50)
elif size>=2**40:
return _("%.2f TiB") % (float(size)/2**40)
elif size>=2**30:
return _("%.2f GiB") % (float(size)/2**30)
elif size>=2**20:
return _("%.2f MiB") % (float(size)/2**20)
elif size>=2**10:
return _("%.2f KiB") % (float(size)/2**10)
else:
return _("%i B") % size
def _pbar_update(self):
"""Update progress bar position, create the bar if it doesn't exist"""
assert(self.progress)
try:
if self.pbar.maxval != self.bytes_total:
self.pbar.maxval = self.bytes_total
self.pbar.widgets[0] = _("Copying %s") % self.__get_string_size(self.bytes_total)
except AttributeError:
if not self.bytes_total:
#No progress bar if the files have a null size
return
self.pbar = ProgressBar(self.bytes_total,[_("Copying %s") % self.__get_string_size(self.bytes_total)," ",Percentage()," ",Bar()," ",FileTransferSpeed()," ",ETA()])
self.pbar.start()
self.pbar.update(self.bytes_copied)
def __pbar_finish(self):
"""Mark the progression as finished"""
assert(self.progress)
try:
self.pbar.finish()
except AttributeError:
pass
11 years ago
def __sourcesSaving(self,options,args):
"""Manage saving/loading/deleting etc of sources files
@param options: options as parsed from command line
@param args: args parsed from command line"""
if options.sources_save or options.sources_load\
or options.sources_list or options.sources_full_list\
or options.sources_del or options.sources_replace:
try:
with open(os.path.expanduser(const_SAVED_LIST),'r') as saved_fd:
saved_files = pickle.load(saved_fd)
except:
saved_files={}
if options.sources_del:
if not saved_files.has_key(options.sources_del):
error(_("No saved sources with this name, check existing names with --sources-list"))
else:
del saved_files[options.sources_del]
with open(os.path.expanduser(const_SAVED_LIST),'w') as saved_fd:
pickle.dump(saved_files,saved_fd)
if not args:
exit(0)
if options.sources_list or options.sources_full_list:
info(_('Saved sources:'))
sources = saved_files.keys()
sources.sort()
for source in sources:
info("\t[%s]" % source)
if options.sources_full_list:
for filename in saved_files[source]:
info("\t\t%s" % filename)
info("---\n")
if not args:
exit(0)
if options.sources_save or options.sources_replace:
if saved_files.has_key(options.sources_save) and not options.sources_replace:
error(_("There is already a saved sources with this name, skipping --sources-save"))
else:
if len(args)>1:
saved_files[options.sources_save] = map(os.path.abspath,args[:-1])
with open(os.path.expanduser(const_SAVED_LIST),'w') as saved_fd:
pickle.dump(saved_files,saved_fd)
if options.sources_load:
if not saved_files.has_key(options.sources_load):
error(_("No saved sources with this name, check existing names with --sources-list"))
else:
saved_args = saved_files[options.sources_load]
saved_args.reverse()
for arg in saved_args:
args.insert(0,arg)
def parseArguments(self, full_args=sys.argv[1:], source_dir = os.getcwd()):
"""Parse arguments and add files to queue
@param full_args: list of arguments strings (without program name)
@param source_dir: path from where the arguments come, as given by os.getcwd()
@return: a tuple (boolean, message) where the boolean is the success of the arguments
validation, and message is the error message to print when necessary"""
11 years ago
_usage="""
%(prog)s [options] FILE DEST
%(prog)s [options] FILE1 [FILE2 ...] DEST-DIR
11 years ago
%(prog)s --help for options list
11 years ago
"""
for idx in range(len(full_args)):
full_args[idx] = full_args[idx].encode('utf-8')
parser = ArgumentParser(usage=_usage)
11 years ago
parser.add_argument("-r", "--recursive", action="store_true", default=False,
help=_("copy directories recursively"))
parser.add_argument("-f", "--force", action="store_true", default=False,
help=_("force overwriting of existing files"))
parser.add_argument("--preserve", action="store", default='',
help=_("preserve the specified attributes"))
parser.add_argument("-L", "--dereference", action="store_true", default=False,
help=_("always follow symbolic links in sources"))
parser.add_argument("-P", "--no-dereference", action="store_false", dest='dereference',
help=_("never follow symbolic links in sources"))
#parser.add_argument("--no-unicode-fix", action="store_false", dest='unicode_fix', default=True,
# help=_("don't fix name encoding errors")) #TODO
parser.add_argument("--fs-fix", choices = const_FS_FIX, dest='fs_fix', default='auto',
help=_("fix filesystem name incompatibily (default: auto)"))
parser.add_argument("--no-fs-fix", action="store_true", dest='no_fs_fix', default=False,
help=_("same as --fs-fix=no (overrides --fs-fix)"))
parser.add_argument("--no-progress", action="store_false", dest="progress", default=True,
help=_("deactivate progress bar"))
parser.add_argument("-v", "--verbose", action="store_true", default=False,
help=_("Show what is currently done"))
parser.add_argument("-V", "--version", action="version", version=ABOUT)
group_saving = parser.add_argument_group("sources saving")
group_saving.add_argument("--sources-save", action="store",
help=_("Save source arguments"))
group_saving.add_argument("--sources-replace", action="store",
help=_("Save source arguments and replace memory if it already exists"))
group_saving.add_argument("--sources-load", action="store",
help=_("Load source arguments"))
group_saving.add_argument("--sources-del", action="store",
help=_("delete saved sources"))
group_saving.add_argument("--sources-list", action="store_true", default=False,
help=_("List names of saved sources"))
group_saving.add_argument("--sources-full-list", action="store_true", default=False,
help=_("List names of saved sources and files in it"))
parser.add_argument_group(group_saving)
(options, args) = parser.parse_known_args()
options.directdir = False #True only in the special case: we are copying a dir and it doesn't exists
#options check
if options.progress and not pbar_available:
warning (_("Progress bar is not available, deactivating"))
options.progress = self.progress = False
else:
self.progress = options.progress
if options.verbose:
logging.getLogger().setLevel(logging.DEBUG)
if options.no_fs_fix:
options.fs_fix = 'no'
if len(options.preserve):
preserve = set(options.preserve.split(','))
if not preserve.issubset(const_PRESERVE):
error (_("Invalid --preserve value\nvalid values are:"))
for value in const_PRESERVE:
error('- %s' % value)
exit(1)
else:
options.preserve = preserve
else:
options.preserve=set()
self.__sourcesSaving(options, args)
if len(args) == 2: #we check special cases
src_path = os.path.abspath(os.path.expanduser(args[0]))
dest_path = os.path.abspath(os.path.expanduser(args[1]))
if os.path.isdir(src_path):
options.dest_file = None #we are copying a dir, this options is for files only
if not os.path.exists(dest_path):
options.directdir = True #dest_dir doesn't exist, it's the directdir special case
elif not os.path.exists(dest_path) or os.path.isfile(dest_path):
options.dest_file = dest_path
args[1] = os.path.dirname(dest_path)
else:
options.dest_file = None
else:
options.dest_file = None
#if there is an other instance of gcp, we send options to it
11 years ago
if not self._main_instance:
info (_("There is already one instance of %s running, pluging to it") % NAME_SHORT)
#XXX: we have to serialize data as dbus only accept valid unicode, and filenames
# can have invalid unicode.
return self.gcp_main.addArgs(pickle.dumps(os.getcwd()),pickle.dumps(full_args))
11 years ago
else:
if len(args) < 2:
_error_msg = _("Wrong number of arguments")
return (False, _error_msg)
debug(_("adding args to gcp: %s") % args)
self.__checkArgs(options, source_dir, args)
if not self.__launched:
self.journal = Journal()
GObject.idle_add(self.__copyNextFile)
self.__launched = True
return (True,'')
11 years ago
def _userInterruption(self):
info(_("User interruption: good bye"))
exit(1)
11 years ago
def go(self):
"""Launch main loop"""
self.loop = GObject.MainLoop()
11 years ago
try:
self.loop.run()
except KeyboardInterrupt:
self._userInterruption()
11 years ago
if __name__ == "__main__":
gcp = GCP()
success,message = gcp.parseArguments()
if not success:
error(message)
exit(1)
11 years ago
if gcp._main_instance:
gcp.go()
if gcp.journal.failed:
exit(1)
10 years ago
if gcp.journal.partial:
exit(2)
11 years ago