Changeset 3251 for WAeUP_SRP/base/WAeUPTool.py
- Timestamp:
- 28 Feb 2008, 09:52:42 (17 years ago)
- File:
-
- 1 edited
-
WAeUP_SRP/base/WAeUPTool.py (modified) (4 diffs)
Legend:
- Unmodified
- Added
- Removed
-
WAeUP_SRP/base/WAeUPTool.py
r3248 r3251 1481 1481 if import_method is None: 1482 1482 error_string += "no %(import_mode)s " % vars() 1483 elif (import_mode in importer.required_modes and 1483 elif (import_mode in importer.required_modes and 1484 1484 not set(importer.required_keys[import_mode]).issubset(set(item.keys()))): 1485 1485 diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys())) … … 1523 1523 pending.append(mapping) 1524 1524 if not pending_only: 1525 log_list += "record added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(),1525 log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(), 1526 1526 else: 1527 1527 already_in += 1 … … 1530 1530 imported_count += 1 1531 1531 imported += mapping, 1532 log_list += "record imported and added to %(imported_fn)s , %(data_string)s" % vars(),1532 log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(), 1533 1533 if log_list: 1534 1534 time_till_now = time.time() - elapse … … 1583 1583 ###) 1584 1584 1585 def d1402_importData(self,filename,name,edit=False,bypass_queue_catalog=False): ###( 1586 """load data from CSV values""" 1587 import transaction 1588 import random 1589 member = self.portal_membership.getAuthenticatedMember() 1590 logger = logging.getLogger('WAeUPTool.importData') 1591 current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") 1592 import_date = DateTime.DateTime().strftime("%d/%m/%y %H:%M:%S") 1593 students_folder = self.portal_url.getPortalObject().campus.students 1594 start = True 1595 tr_count = 0 1596 total_imported = 0 1597 total_not_imported = 0 1598 total = 0 1599 pending_only = False 1600 pend_str = '_pending' 1601 if name.endswith(pend_str): 1602 pending_only = True 1603 name = name[:-len(pend_str)] 1604 iname = "import_%s" % name 1605 if name in ('application','course_result',): 1606 commit_after = 2000 1607 else: 1608 commit_after = 100 1609 stool = getToolByName(self, 'portal_schemas') 1610 ltool = getToolByName(self, 'portal_layouts') 1611 schema = stool._getOb(iname) 1612 if schema is None: 1613 em = 'No such schema %s' % iname 1614 logger.error('No such schema %s' % iname) 1615 return em 1616 layout = ltool._getOb(iname) 1617 if layout is None: 1618 em = 'No such layout %s' % iname 1619 logger.error(em) 1620 return em 1621 validators = {} 1622 for widget in layout.keys(): 1623 validators[widget] = layout[widget].validate 1624 mode = "create" 1625 if edit: 1626 if filename.endswith('_toDelete'): 1627 mode = "delete" 1628 else: 1629 mode = "edit" 1630 importer_name = "mass_%(mode)s_%(name)s" % vars() 1631 importer = getattr(self, '%s' % importer_name,None) 1632 if importer is None: 1633 em = 'No importer function %s' % importer_name 1634 logger.error(em) 1635 return em 1636 pending_fn = "%s/import/%ss_pending.csv" % (i_home,name) 1637 pending_imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current) 1638 if pending_only: 1639 import_source_fn = pending_fn 1640 imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current) 1641 not_imported_fn = "%s/import/%ss_pending_not_imported%s.csv" % (i_home,name,current) 1642 if not os.path.exists(pending_fn): 1643 em = 'No %s' % os.path.split(pending_fn) 1644 return em 1645 else: 1646 import_source_fn = "%s/import/%s.csv" % (i_home,filename) 1647 imported_fn = "%s/import/%s_imported%s.csv" % (i_home,filename,current) 1648 not_imported_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current) 1649 if not os.path.exists(import_source_fn): 1650 em = 'No %s' % os.path.split(import_soure_fn) 1651 return em 1652 attrs = csv.reader(open(import_source_fn,"rb")).next() 1653 import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore') 1654 or k.strip() == 'Error')] 1655 diff2schema = set(import_keys).difference(set(schema.keys())) 1656 diff2layout = set(import_keys).difference(set(layout.keys())) 1657 if diff2schema and diff2schema != set(['id',]): 1658 msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema) 1659 return msg 1660 if mode in ("create","edit"): 1661 required_keys = [layout.getIdUnprefixed(id) 1662 for id,widget in layout.objectItems() 1663 if widget.is_required] 1664 if not set(required_keys).issubset(set(import_keys)): 1665 diff2import = set(required_keys).difference(set(import_keys)) 1666 msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import) 1667 return msg 1668 # 1669 # not_imported 1670 # 1671 not_imported_keys = import_keys + ["Error",] 1672 not_imported_file = open(not_imported_fn,"w") 1673 not_imported_csv_writer = csv.DictWriter(not_imported_file, 1674 not_imported_keys, 1675 extrasaction='ignore') 1676 mapping = dict((k,k) for k in not_imported_keys) 1677 not_imported_csv_writer.writerow(mapping) 1678 not_imported = [] 1679 # 1680 # imported 1681 # 1682 imported_keys = import_keys[:] 1683 if 'id' not in imported_keys: 1684 imported_keys.insert(0,"id") 1685 imported_file = open(imported_fn,"w") 1686 imported_csv_writer = csv.DictWriter(imported_file, 1687 imported_keys, 1688 extrasaction='ignore') 1689 mapping = dict((k,k) for k in imported_keys) 1690 imported_csv_writer.writerow(mapping) 1691 imported = [] 1692 # 1693 # pending 1694 # 1695 pending_records = {} 1696 imports_pending = 0 1697 total_pending_imported = 0 1698 total_pending = 0 1699 format = ','.join(['"%%(%s)s"' % fn for fn in import_keys]) 1700 format_error = format + ',"%(Error)s"' 1701 #format = '"%(id)s",'+ format 1702 imported = [] 1703 if name in ('verdict','course_result',): 1704 #pending_keys = imported_keys[:] 1705 sname = "%s_pending" % iname 1706 pending_schema = stool._getOb(sname) 1707 if schema is None: 1708 em = 'No such pending_schema %s' % sname 1709 logger.error('No such schema %s' % sname) 1710 return em 1711 pending_keys = pending_schema.keys() 1712 pending_keys += "Error", 1713 if 'id' not in pending_keys: 1714 pending_keys.insert(0,'id') 1715 pending_records = {} 1716 if os.path.exists(pending_fn): 1717 pending_imports_file = open(pending_fn) 1718 pending_imports = csv.DictReader(pending_imports_file, 1719 fieldnames=pending_keys) 1720 if pending_imports: 1721 pending_records = dict((row['id'],row) for row in pending_imports 1722 if row['id'] != 'id') 1723 pending_imports_file.close() 1724 new_imports_pending = 0 1725 pending_imported = [] 1726 if pending_only: 1727 pending_imported_ids = [] 1728 pending_imported = [] 1729 for record in pending_records.values(): 1730 item = {} 1731 for k,v in record.items(): 1732 if v: 1733 item[k] = v 1734 results = importer(item) 1735 id = results[0] 1736 error = results[1] 1737 is_pending = len(results) == 3 1738 if is_pending: 1739 continue 1740 del item['Error'] 1741 msg = ";".join(["%s : %s" % (k,v) for k,v in item.items()]) 1742 logger.info("imported from %(pending_fn)s %(msg)s" % vars()) 1743 pending_imported.append(item) 1744 pending_imported_ids += id, 1745 total_pending_imported = len(pending_imported_ids) 1746 logger.info("imported %d pending imports" % len(pending_imported_ids)) 1747 for id in pending_imported_ids: 1748 if id: 1749 del pending_records[id] 1750 else: 1751 logger.info("tried to delete record with empty id") 1752 pending_imports_file = open(pending_fn,'w') 1753 pending_csv_writer = csv.DictWriter(pending_imports_file, 1754 pending_keys, 1755 extrasaction='ignore') 1756 mapping = dict((k,k) for k in pending_keys) 1757 pending_csv_writer.writerow(mapping) 1758 if len(pending_imported) > 0: 1759 pending_imported_file = open(pending_imported_fn,'w') 1760 pending_imported_csv_writer = csv.DictWriter(pending_imported_file, 1761 pending_keys, 1762 extrasaction='ignore') 1763 pending_imported_csv_writer.writerow(dict((k,k) for k in imported_keys)) 1764 pending_imported_csv_writer.writerows(pending_imported) 1765 # 1766 if pending_only: 1767 items = [] 1768 else: 1769 try: 1770 items = csv.DictReader(open(import_source_fn,"rb")) 1771 except: 1772 msg = 'Error reading %s.csv' % filename 1773 logger.error(msg) 1774 return msg 1775 not_imported = [] 1776 pending_keys = pending_records.keys()[:] 1777 for item in items: 1778 item = dict((k.strip(),l.strip()) for (k,l) in item.items()) 1779 if start: 1780 start = False 1781 adapters = [MappingStorageAdapter(schema, item)] 1782 logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname)) 1783 dm = DataModel(item, adapters,context=self) 1784 ds = DataStructure(data=item,datamodel=dm) 1785 error_string = "" 1786 total += 1 1787 error_count = 0 1788 for k in import_keys: 1789 if not validators[k](ds,mode=mode): 1790 if error_count: 1791 error_string += ' ++ ' 1792 error_string += "%s: %s" % (k,self.translation_service(ds.getError(k), 1793 ds.getErrorMapping(k))) 1794 error_count += 1 1795 if error_string: 1796 item['Error'] = error_string 1797 not_imported.append(item) 1798 total_not_imported += 1 1799 continue 1800 temp_item = item.copy() 1801 temp_item.update(dm) 1802 #id,error = importer(temp_item) 1803 results = importer(temp_item) 1804 id = results[0] 1805 error = results[1] 1806 is_pending = len(results) == 3 1807 if is_pending: 1808 temp_item = results[2] 1809 temp_item['Error'] = error 1810 msg = format_error % temp_item 1811 #if id not in pending_records.keys(): 1812 if id not in pending_keys: 1813 temp_item['id'] = id 1814 pending_records[id] = temp_item 1815 logger.info("%(id)s added to pending %(msg)s" % vars()) 1816 pending_keys += id, 1817 new_imports_pending += 1 1818 else: 1819 logger.info("%(id)s already in pending %(msg)s" % vars()) 1820 continue 1821 elif error: 1822 item['Error'] = error 1823 not_imported.append(item) 1824 total_not_imported += 1 1825 continue 1826 item = temp_item 1827 item['id'] = id 1828 imported.append(item) 1829 tr_count += 1 1830 total_imported += 1 1831 msg = format % item 1832 logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(msg)s" % vars()) 1833 if total and not total % commit_after: 1834 transaction.commit() 1835 if len(imported) > 0: 1836 imported_csv_writer.writerows(imported) 1837 imported = [] 1838 if len(not_imported) > 0: 1839 not_imported_csv_writer.writerows(not_imported) 1840 not_imported = [] 1841 if len(pending_records) > 0: 1842 pending_csv_writer.writerows(pending_records.values()) 1843 total_pending += len(pending_records) 1844 pending_records = {} 1845 msg = '%d transactions committed\n' % (tr_count) 1846 regs = [] 1847 logger.info(msg) 1848 tr_count = 0 1849 if len(imported) > 0: 1850 imported_csv_writer.writerows(imported) 1851 if len(not_imported) > 0: 1852 not_imported_csv_writer.writerows(not_imported) 1853 if len(pending_records) > 0: 1854 pending_csv_writer.writerows(pending_records.values()) 1855 total_pending += len(pending_records) 1856 import_source_fn = os.path.split(import_source_fn)[1] 1857 pending_fn = os.path.split(pending_fn)[1] 1858 msg = "Finished importing in %(mode)s mode from %(import_source_fn)s: " % vars() 1859 msg += "%(total_imported)d imported, %(total_not_imported)d not imported, " % vars() 1860 if total_pending: 1861 if pending_only: 1862 msg += "%(new_imports_pending)d pending added, %(total_pending_imported)d pending imported " % vars() 1863 else: 1864 msg += "%(new_imports_pending)d pending added " % vars() 1865 msg += "(of total %(total)d), " % vars() 1866 msg += "%(total_pending)d total pending in %(pending_fn)s " % vars() 1867 logger.info(msg) 1868 return msg 1869 ###) 1585 1870 1586 1871 1587 security.declareProtected(ModifyPortalContent,"moveImagesToFS")###(
Note: See TracChangeset for help on using the changeset viewer.
