[8523] | 1 | ## |
---|
| 2 | ## This script is called without parameters |
---|
| 3 | ## |
---|
| 4 | ## It copies media files from an old SRP folder over to a Kofa |
---|
| 5 | ## instance media folder. It does not remove copied folders. |
---|
| 6 | ## |
---|
| 7 | ## Configuration is done below. |
---|
| 8 | ## |
---|
| 9 | ## At end a list of dirs that were copied or are empty is returned. |
---|
| 10 | ## |
---|
| 11 | ## Please note that for changing groups of files/directories, you |
---|
| 12 | ## normally have to be root. This script is therefore normally run with |
---|
| 13 | ## sudo. |
---|
| 14 | ## |
---|
| 15 | |
---|
| 16 | ## ###################################################################### |
---|
| 17 | ## CONFIGURATION |
---|
| 18 | ## |
---|
| 19 | |
---|
| 20 | ## The folder where all source docs can be found |
---|
| 21 | #SRC_DIR = "/zope/instances/aaue-images" |
---|
[9365] | 22 | SRC_DIR = "/tmp/migration/src" |
---|
[8523] | 23 | |
---|
| 24 | ## The students folder in Kofa where files should go to |
---|
| 25 | #DST_DIR = "/kofa/aaue/var/datacenter/media/students" |
---|
[9365] | 26 | DST_DIR = "/tmp/migration/dst" |
---|
[8523] | 27 | |
---|
| 28 | ## The Ids of new students to search for in old portal. |
---|
| 29 | NEW_IDS_CSV = "StudentIds.csv" |
---|
| 30 | |
---|
[9362] | 31 | ## The ids of students that will will get a complete new number, not |
---|
| 32 | ## an updated one. Must contain a mapping of old ids to new ids. |
---|
| 33 | ## If no such update should be performed, please set the constant to None. |
---|
| 34 | ID_MAP_CSV = None |
---|
| 35 | ID_MAP_CSV = "id_mapping.csv" |
---|
| 36 | |
---|
[8523] | 37 | ## Permissions to be set on new files/dirs. Set OWNER and/or GROUP to |
---|
| 38 | ## None to leave them unchanged after creation. |
---|
| 39 | UMASK = 0664 |
---|
| 40 | OWNER = 'uli' |
---|
| 41 | GROUP = 'nogroup' |
---|
| 42 | |
---|
| 43 | ## |
---|
| 44 | ## CONFIGURATION END |
---|
| 45 | ## ###################################################################### |
---|
| 46 | |
---|
| 47 | import csv |
---|
| 48 | import os |
---|
| 49 | import shutil |
---|
| 50 | import sys |
---|
| 51 | from grp import getgrnam |
---|
| 52 | from pwd import getpwnam |
---|
| 53 | |
---|
| 54 | OWNER_ID = OWNER and getpwnam(OWNER)[2] or -1 |
---|
| 55 | GRP_ID = GROUP and getgrnam(GROUP)[2] or -1 |
---|
| 56 | DIR_UMASK = UMASK | 0111 # directories need extra x-permission |
---|
| 57 | |
---|
| 58 | def set_perms(path): |
---|
| 59 | # set permissions and ownership for path |
---|
| 60 | if os.path.isdir(path): |
---|
| 61 | os.chmod(path, DIR_UMASK) |
---|
| 62 | else: |
---|
| 63 | os.chmod(path, UMASK) |
---|
| 64 | os.chown(path, OWNER_ID, GRP_ID) |
---|
| 65 | return |
---|
| 66 | |
---|
| 67 | def new_folder_name(stud_id): |
---|
| 68 | # compute new folder name from stud_id (old stud_ids only) |
---|
| 69 | num = int(stud_id[1:]) |
---|
| 70 | num = num / 10000 * 10 |
---|
| 71 | return '%05d' % num |
---|
| 72 | |
---|
| 73 | def create_path(path): |
---|
| 74 | # create path with subdirs, if it does not exist (completely) |
---|
| 75 | if os.path.exists(path): |
---|
| 76 | return |
---|
| 77 | parent = os.path.dirname(path) |
---|
| 78 | if not os.path.exists(parent): |
---|
| 79 | # create parent first |
---|
| 80 | create_path(parent) |
---|
| 81 | print "CREATE PATH ", path |
---|
| 82 | os.mkdir(path) |
---|
| 83 | set_perms(path) |
---|
| 84 | return |
---|
| 85 | |
---|
[8924] | 86 | def copy_file(file_src, file_dst): |
---|
[9362] | 87 | create_path(os.path.dirname(file_dst)) |
---|
[8523] | 88 | print "COPY FILE: %s -> %s" % (file_src, file_dst) |
---|
| 89 | shutil.copyfile(file_src, file_dst) |
---|
| 90 | set_perms(file_dst) |
---|
| 91 | return |
---|
| 92 | |
---|
[9362] | 93 | def get_new_old_id_mapping(): |
---|
| 94 | """Returns a dict mapping from _new_ ids to old (SRP) ids. |
---|
| 95 | |
---|
| 96 | The dict is read from ID_MAP_CSV file. If this var is set to |
---|
| 97 | ``None`` an empty dict is returned. The ID_MAP_CSV contains only |
---|
| 98 | the student ids of those students, for which the standard method |
---|
| 99 | (new_id=CHAR+old_id) does not work. |
---|
| 100 | """ |
---|
| 101 | if ID_MAP_CSV is None: |
---|
| 102 | return {} |
---|
| 103 | if not os.path.isfile(ID_MAP_CSV): |
---|
| 104 | raise IOError( |
---|
| 105 | "No such file for mapping new to old ids: %s" % ID_MAP_CSV) |
---|
| 106 | result = dict() |
---|
| 107 | reader = csv.DictReader(open(ID_MAP_CSV, 'rb')) |
---|
| 108 | for row in reader: |
---|
| 109 | result[row['new_id']] = row['student_id'] |
---|
| 110 | return result |
---|
| 111 | |
---|
| 112 | # special ids not handled in common way |
---|
| 113 | new_to_old_map = get_new_old_id_mapping() |
---|
[8523] | 114 | removable_dirs = [] |
---|
| 115 | reader = csv.DictReader(open(NEW_IDS_CSV, 'rb')) |
---|
| 116 | for row in reader: |
---|
| 117 | new_stud_id = row['student_id'] |
---|
[9362] | 118 | stud_id = new_to_old_map.get(new_stud_id[1:], new_stud_id[1:]) |
---|
[8523] | 119 | src_folder = os.path.join(SRC_DIR, stud_id[0], stud_id) |
---|
[9365] | 120 | dst_folder = os.path.join( |
---|
| 121 | DST_DIR, new_folder_name(new_stud_id[1:]), new_stud_id) |
---|
[8523] | 122 | if not os.path.exists(src_folder): |
---|
| 123 | print "No old data for %s. Skipping." % stud_id |
---|
| 124 | continue |
---|
| 125 | if os.path.exists(dst_folder): |
---|
| 126 | print "Destination folder exists already: %s. Skipping" % (dst_folder) |
---|
| 127 | continue |
---|
| 128 | removable_dirs.append(src_folder) |
---|
| 129 | src_files = os.listdir(src_folder) |
---|
| 130 | if len(src_files) == 0: |
---|
| 131 | print "Empty source folder for %s. Skipping." % stud_id |
---|
| 132 | continue |
---|
| 133 | for name in src_files: |
---|
| 134 | file_src = os.path.join(src_folder, name) |
---|
| 135 | file_dst = os.path.join(dst_folder, name.replace(stud_id, new_stud_id)) |
---|
| 136 | copy_file(file_src, file_dst) |
---|
| 137 | |
---|
| 138 | print "DIRS TO REMOVE: " |
---|
| 139 | for name in removable_dirs: |
---|
| 140 | print name |
---|