source: WAeUP_SRP/trunk/skins/waeup_utilities/ti_441_resolve_part2.py @ 17305

## Script (Python) "ti_441_resolve_part2"
##bind container=container
##bind context=context
##bind namespace=
##bind script=script
##bind subpath=traverse_subpath
##parameters=
##title=
##
# $Id: fix_lgas.py 2979 2008-01-05 15:01:33Z henrik $
"""
"""
try:
    from Products.zdb import set_trace
except:
    def set_trace():
        pass


mtool = context.portal_membership
member = mtool.getAuthenticatedMember()
if str(member) not in ('admin','joachim'):
    return


import logging
import DateTime
logger = logging.getLogger('Skins.ti_441_resolve_part2')
from Products.AdvancedQuery import Eq, Between, Le,In
aq_portal = context.portal_catalog_real.evalAdvancedQuery
aq_students = context.students_catalog.evalAdvancedQuery
students_folder = context.portal_url.getPortalObject().campus.students

request = context.REQUEST
session = request.SESSION
response = request.RESPONSE
setheader = request.RESPONSE.setHeader
def rwrite(s):
    response.setHeader('Content-type','text/html; charset=ISO-8859-15')
    response.write("%s<br>\n\r" % s)

# def getLGAs():
#     voc = getattr(context.portal_vocabularies,'local_gov_areas')
#     states = []
#     lgas  = []
#     d = {}
#     for k,v in voc.items():
#         parts = v.split(' / ')
#         if len(parts) == 1:
#             state = parts[0].lower()
#             lga = ""
#         elif len(parts) == 2:
#             state = parts[0].lower()
#             lga = "_".join(parts[1].lower().split())
#         else:
#             continue
#         if state not in states:
#             states.append(state)
#         if lga not in lgas:
#             lgas.append(lga)
#         d[k] = v
#     return (d,states,lgas)

#difference = context.waeup_tool.difference
re_split = context.waeup_tool.re_split
#logger.info('')
student_records = context.students_catalog()
total = len(student_records)
logger.info("found %d students" % total)
count = 0
correctable = wrong = 1
commit_after = 100
logger.info('started to fix %d records in students_catalog' % total)
mapping = context.waeup_tool.getStatesLgas()
states = mapping['states']
lgas = mapping['lgas']
words_dict = mapping['word_dict']
d = {}
not_found_lgas = []
not_found_count = {}
rename_lgas = {}
# rename_lgas['abia_owo'] = 'abia_owo'
# rename_lgas['abia_ughelli-nort'] = 'abia_ughelli-nort'
# rename_lgas['akwa ibom_urue-offong-oruk'] = 'akwa ibom_urue-offong-oruk'
# rename_lgas['anambra_anambr'] = 'anambra_anambr'
# rename_lgas['anambra_fufor'] = 'anambra_fufor'
# rename_lgas['bayelsa_borip'] = 'bayelsa_borip'
# rename_lgas['bayelsa_kolokuma-opkum'] = 'bayelsa_kolokuma-opkum'
# rename_lgas['cross river_aboa/odua'] = 'cross river_aboa/odua'
# rename_lgas['delta_ihial'] = 'delta_ihial'
# rename_lgas['delta_ik'] = 'delta_ik'
# rename_lgas['delta_ikpemil'] = 'delta_ikpemil'
# rename_lgas['delta_ile-oluji-okeigb'] = 'delta_ile-oluji-okeigb'
# rename_lgas['delta_nnewi-sout'] = 'delta_nnewi-sout'
# rename_lgas['delta_ored'] = 'delta_ored'
# rename_lgas['delta_osogb'] = 'delta_osogb'
# rename_lgas['delta_owan-eas'] = 'delta_owan-eas'
# rename_lgas['delta_uhunmwond'] = 'delta_uhunmwond'
# rename_lgas['edo_abakalik'] = 'edo_abakalik'
# rename_lgas['edo_ihial'] = 'edo_ihial'
# rename_lgas['edo_isoko-sout'] = 'edo_isoko-sout'
# rename_lgas['edo_nkanu-wes'] = 'edo_nkanu-wes'
# rename_lgas['edo_oshimili-nort'] = 'edo_oshimili-nort'
# rename_lgas['edo_uden'] = 'edo_uden'
# rename_lgas['edo_ughelli-nort'] = 'edo_ughelli-nort'
# rename_lgas['edo_ukwuan'] = 'edo_ukwuan'
# rename_lgas['edo_uvwe'] = 'edo_uvwe'
# rename_lgas['edo_uvwi'] = 'edo_uvwi'
# rename_lgas['edo_uy'] = 'edo_uy'
# rename_lgas['ekiti_ughelli-sout'] = 'ekiti_ughelli-sout'
# rename_lgas['imo_ahoada-wes'] = 'imo_ahoada-wes'
# rename_lgas['kogi_owan-wes'] = 'kogi_owan-wes'
# rename_lgas['kwara_orhionmwo'] = 'kwara_orhionmwo'
# rename_lgas['ogun_ijer'] = 'ogun_ijer'

for student_record in student_records:
    # skip record without lga value
    count += 1
    if not student_record.lga:
        continue
    # if student_record.lga.find(' ') < 0:
    #     continue
    if student_record.lga.startswith('no state'):
        #logger.info("found invalid lga %s of %s" % (student_record.lga,student_record.id))
        continue
    #rwrite("%s: %s" % (student_record.id,student_record.lga))
    if student_record.lga in rename_lgas.keys():
        correctable += 1
        d[student_record.id] = rename_lgas[student_record.lga]
        if len(d) and not len(d)  % commit_after:
            logger.info("found %d to correct state/lga combinations of %d so far" % (len(d),count))
        continue
    words = student_record.lga 
    if len(words.split(' / ')) == 2:
        words = words.replace(' / ',' ')
    state_lga = context.waeup_tool.findLga(words,words_dict)
    if state_lga:
        if state_lga != student_record.lga:
            correctable += 1
            msg = "found %s is %s for %s " % (state_lga,student_record.lga,student_record.id)
            #rwrite(msg)
            d[student_record.id] = state_lga
            if len(d) and not len(d)  % commit_after:
                logger.info("found %d to correct state/lga combinations of %d so far" % (len(d),count))
        else:
            msg = "already corrected %s : %s for %s " % (state_lga,student_record.lga,student_record.id)
    else:
        wrong += 1
        msg = "no '%s' for %s" % (words,student_record.id)
        if len(words) > 2:
            if words not in not_found_lgas:
                not_found_lgas += words,
                not_found_count[words] = 1
            else:
                nfc = not_found_count[words] 
                nfc += 1
                not_found_count[words] = nfc 
        #rwrite(msg)
    # if count > 2000:
    #     break
not_found_lgas.sort()
for lga in not_found_lgas:
    msg = "not found %s count: %d" % (lga,not_found_count[lga])
    logger.info(msg)
to_edit = len(d)
logger.info("found %d correctable state/lga combinations, not correctable %d total %d" % (correctable,
                                                                                          wrong,
                                                                                          count))
edited = 1
for student_id,lga in d.items():
    msg = "set clearance.lga to %s for %s" % (lga,student_id)
    getattr(getattr(students_folder,student_id),'clearance').getContent().edit(mapping={'lga':lga})
    logger.info(msg)
    edited += 1
    if edited and not edited  % commit_after:
        context.waeup_tool.doCommit()
        logger.info("Committing %d,  %d of %d" % (commit_after,edited,to_edit))
logger.info('finished, %d checked, %d edited' % (count,
                                               edited,))