Changeset 13202


Ignore:
Timestamp:
21 Aug 2015, 13:22:11 (9 years ago)
Author:
uli
Message:

Reorder functions.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • main/waeup.uniben/trunk/src/waeup/uniben/scripts.py

    r13195 r13202  
    8181
    8282
    83 def export():
    84     """Main function.
    85     """
    86     closables = init_dbs()
    87     conn = closables[0].open()
    88     uni = get_university(conn)
    89     studs = uni['students']
    90 
    91     parts = partition(studs)
    92     print(parts)
    93 
    94     #parts = [(0, 0, 100), ]
    95     #parts = [(0, 0, 4999), (1, 5000, 9999), (2, 10000, 14999)]
    96     #parts = [(0, 0, 14999), ]
    97     #parts = [(0, 0, 160000), ]
    98 
    99     workdir = tempfile.mkdtemp()
    100 
    101     t1 = time.time()
    102     conn.close()  # every loop will reopen the connection
    103     for p_num, start, end in parts:
    104         conn = closables[0].open()
    105         uni = get_university(conn)
    106         studs = uni['students']
    107         curr_path = os.path.join(workdir, "myexport%s.csv" % p_num)
    108         export_part(studs, p_num, start, end, curr_path)
    109         conn.close()
    110     print("Result in %s" % workdir)
    111     t2 = time.time()
    112     print("Elapsed: %s secs" % (t2 - t1))
    113 
    114 
    115 def export_part(container, part_num, start, end, path):
    116     import psutil  # late import. Don't want to make it waeup.uniben req.
    117     PROC = psutil.Process(os.getpid())
    118     gc.collect()
    119     mem1 = PROC.get_memory_info().rss
    120     print("  Export %s-%s to %s (mem: %s)" % (start, end, path, mem1))
    121     tp1 = time.time()
    122     exporter = getUtility(ICSVExporter, name="students")
    123     exporter.export(get_all_students(container, start, end), path)
    124     tp2 = time.time()
    125     mem2 = PROC.get_memory_info().rss
    126     print("  Done (%s secs, mem: %s)" % (tp2 - tp1, mem2))
    127 
    128 
    12983def get_all_students(container, cnt_from=0, cnt_to=0):
    13084    cnt = 0
     
    165119        (idx, start, min(start + part_size - 1, num - 1))
    166120        for idx, start in enumerate(range(0, num, part_size))]
     121
     122
     123def export_part(container, part_num, start, end, path):
     124    import psutil  # late import. Don't want to make it waeup.uniben req.
     125    PROC = psutil.Process(os.getpid())
     126    gc.collect()
     127    mem1 = PROC.get_memory_info().rss
     128    print("  Export %s-%s to %s (mem: %s)" % (start, end, path, mem1))
     129    tp1 = time.time()
     130    exporter = getUtility(ICSVExporter, name="students")
     131    exporter.export(get_all_students(container, start, end), path)
     132    tp2 = time.time()
     133    mem2 = PROC.get_memory_info().rss
     134    print("  Done (%s secs, mem: %s)" % (tp2 - tp1, mem2))
     135
     136
     137def export():
     138    """Main function.
     139    """
     140    closables = init_dbs()
     141    conn = closables[0].open()
     142    uni = get_university(conn)
     143    studs = uni['students']
     144
     145    parts = partition(studs)
     146    print(parts)
     147
     148    #parts = [(0, 0, 100), ]
     149    #parts = [(0, 0, 4999), (1, 5000, 9999), (2, 10000, 14999)]
     150    #parts = [(0, 0, 14999), ]
     151    #parts = [(0, 0, 160000), ]
     152
     153    workdir = tempfile.mkdtemp()
     154
     155    t1 = time.time()
     156    conn.close()  # every loop will reopen the connection
     157    for p_num, start, end in parts:
     158        conn = closables[0].open()
     159        uni = get_university(conn)
     160        studs = uni['students']
     161        curr_path = os.path.join(workdir, "myexport%s.csv" % p_num)
     162        export_part(studs, p_num, start, end, curr_path)
     163        conn.close()
     164    print("Result in %s" % workdir)
     165    t2 = time.time()
     166    print("Elapsed: %s secs" % (t2 - t1))
Note: See TracChangeset for help on using the changeset viewer.