Ticket #1499: activity-update-incremental.patch
File activity-update-incremental.patch, 14.2 KB (added by dsd, 15 years ago) |
---|
-
model/updater.py
Incrementally update activities Based on earlier work by Aayush Poudel @ OLE Nepal
old new import gtk 27 27 import gobject 28 28 29 29 import locale 30 import logging 30 31 import os 31 32 import os.path 32 33 import socket 34 import shutil 33 35 import sys 34 36 import traceback 35 37 import zipfile 38 import zlib 36 39 from HTMLParser import HTMLParseError 37 40 from urllib2 import HTTPError 38 41 … … import bitfrost.update.actutils as actut 44 47 import bitfrost.update.microformat as microformat 45 48 import bitfrost.util.urlrange as urlrange 46 49 50 _logger = logging.getLogger("updater") 51 47 52 # weak dependency on inhibit_suspend from olpc-update package 48 53 try: 49 54 from bitfrost.update import inhibit_suspend … … def _humanize_size(bytes): 69 74 # TRANSLATORS: download size of updates, e.g. "2.3 MB" 70 75 return locale.format(_("%.1f MB"), bytes / 1024 / 1024) 71 76 77 def _file_crc32(path): 78 fd = open(path, 'rb') 79 crc = 0 80 81 while True: 82 buffer = fd.read(8192) 83 if not buffer: 84 break 85 crc = zlib.crc32(buffer, crc) 86 87 fd.close() 88 return crc 89 90 # FIXME use os.path.relpath when Python2.6 is required 91 def _relpath(path, start): 92 start_list = os.path.abspath(start).split('/') 93 path_list = os.path.abspath(path).split('/') 94 i = len(os.path.commonprefix([start_list, path_list])) 95 rel_list = [".."] * (len(start_list)-i) + path_list[i:] 96 if not rel_list: 97 return start 98 return os.path.join(*rel_list) 99 100 def _extract_from_zip(zip, zinfo, base_dir): 101 fd = open(os.path.join(base_dir, zinfo.filename), 'wb') 102 103 # python-2.5.1 zipfile does not like read() of a 0 byte file 104 if zinfo.file_size > 0: 105 fd.write(zip.read(zinfo.filename)) 106 107 fd.close() 108 72 109 def _svg2pixbuf(icon_data): 73 110 """Convert the given `icon_data` SVG string to a `gtk.gdk.Pixbuf` 74 111 with maximum size 55x55.""" … … class UpdateList(gtk.ListStore): 489 526 self._cancel = True 490 527 491 528 def cancel_download(self): 492 """Asynchronously cancel a `download_selected_updates`operation."""529 """Asynchronously cancel an update operation.""" 493 530 self._cancel = True 494 531 495 532 def _sum_rows(self, row_func): … … class UpdateList(gtk.ListStore): 508 545 return self._sum_rows(lambda r: 1 if 509 546 r[UPDATE_EXISTS] and r[UPDATE_SELECTED] else 0) 510 547 def updates_size(self): 511 """Returns the size (in bytes) of the selected updates available. 548 """Returns the size (in bytes) of the selected updates and new 549 activities available. 512 550 513 551 Updated by `refresh`.""" 514 552 return self._sum_rows(lambda r: r[UPDATE_SIZE] if 515 553 r[UPDATE_EXISTS] and r[UPDATE_SELECTED] else 0) 554 def updates_size_new_only(self): 555 """Returns the size (in bytes) of the selected new activities. 556 557 Updated by `refresh`.""" 558 return self._sum_rows(lambda r: r[UPDATE_SIZE] if 559 r[UPDATE_EXISTS] and r[UPDATE_SELECTED] and 560 r[ACTIVITY_BUNDLE] is None else 0) 561 def updates_size_updates_only(self): 562 """Returns the size (in bytes) of the selected updates available. 563 564 Updated by `refresh`.""" 565 return self._sum_rows(lambda r: r[UPDATE_SIZE] if 566 r[UPDATE_EXISTS] and r[UPDATE_SELECTED] and 567 r[ACTIVITY_BUNDLE] is not None else 0) 516 568 def unselect_all(self): 517 569 """Unselect all available updates.""" 518 570 for row in self: … … class UpdateList(gtk.ListStore): 657 709 if row[IS_HEADER] and row[UPDATE_URL] is not None: 658 710 print >>f, row[UPDATE_URL] 659 711 660 def download_ selected_updates(self, progress_cb=(lambda n, row: None),661 712 def download_new_activities(self, progress_cb=(lambda n, row: None), 713 dir=None): 662 714 """Return a generator giving (row, local filename) pairs for 663 715 each selected update. Caller is responsible for unlinking 664 716 these files when they are through. The `progress_cb` gets a … … class UpdateList(gtk.ListStore): 674 726 if dir is None: dir=actinfo.USER_ACTIVITY_DIR 675 727 if not os.path.isdir(dir): os.makedirs(dir) 676 728 self._cancel = False 677 sizes = [ 0, self.updates_size () ]729 sizes = [ 0, self.updates_size_new_only() ] 678 730 for row in self: 679 731 if self._cancel: return # bail. 680 732 if row[IS_HEADER]: continue 681 733 if not (row[UPDATE_EXISTS] and row[UPDATE_SELECTED]): continue 734 735 # dont handle updates here 736 if row[ACTIVITY_BUNDLE] is not None: continue 737 682 738 def report(cursize, totalsize_): 683 739 if totalsize_ is None: 684 740 progress_cb(None, row) … … class UpdateList(gtk.ListStore): 700 756 sizes[0] += row[UPDATE_SIZE] 701 757 progress_cb(sizes[0] / sizes[1], row) 702 758 759 def do_selected_updates(self, progress_cb): 760 """ 761 Update activities by updating their files incrementally, only 762 downloading the files that have changed. 763 """ 764 765 self._cancel = False 766 sizes = [ 0, self.updates_size_updates_only() ] 767 768 def _progress_cb(n, row): 769 progress_cb((sizes[0] + (n*row[UPDATE_SIZE])) / sizes[1], row) 770 771 for row in self: 772 if self._cancel: return # bail. 773 if row[IS_HEADER]: continue 774 if not (row[UPDATE_EXISTS] and row[UPDATE_SELECTED]): continue 775 if row[ACTIVITY_BUNDLE] is None: continue # updates only 776 777 _logger.debug("incremental update %s" % row[ACTIVITY_ID]) 778 old_path, new_path = self.do_inplace_update(row, _progress_cb) 779 sizes[0] += row[UPDATE_SIZE] 780 progress_cb(sizes[0] / sizes[1], row) 781 yield row, old_path, new_path 782 783 def do_inplace_update(self, row, progress_cb): 784 """ 785 Incrementally update an activity according to the following algorithm: 786 787 1. If the base activity directory has changed, rename the local version 788 2. Delete any local files and directories that are not present in the 789 upstream version 790 3. Download files from the upstream version that are not present 791 locally, or if the local versions differ. 792 4. Atomically update activity.info 793 794 Deletion is done first, because the disk space freed up by deletion 795 may be required for the downloading of files in step 3. 796 797 activity.info update is atomic so that you are never left in the 798 situation where you cannot delete the activity from sugar (or re-run 799 the updater). 800 801 This operation cannot be cancelled. 802 """ 803 try: 804 remote_zip = zipfile.ZipFile(urlrange.urlopen(row[UPDATE_URL], use_cache=False, timeout=30), 'r') 805 except (HTMLParseError, IOError, socket.error): 806 return None, None # network error 807 808 # e.g. /home/olpc/Activities/Foo.activity 809 activity_path = row[ACTIVITY_BUNDLE].get_path().rstrip('/') 810 _logger.debug("activity_path %s" %activity_path) 811 812 # e.g. /home/olpc/Activities 813 activities_dir = os.path.dirname(activity_path) 814 _logger.debug("activities_dir %s" % activities_dir) 815 816 # e.g. Foo.activity (name of activity directory on local disk) 817 local_base_dir = activity_path.split('/')[-1] 818 _logger.debug("local base dir %s" % local_base_dir) 819 820 # e.g. Foo.activity (name of activity directory in remote zipfile) 821 remote_base_dir = actutils.bundle_base_from_zipfile(remote_zip) 822 _logger.debug("remote base dir %s" % remote_base_dir) 823 824 old_path = activity_path 825 new_path = os.path.join(activities_dir, remote_base_dir) 826 827 # last opportunity to safely cancel 828 if self._cancel: 829 return None, None 830 831 if local_base_dir != remote_base_dir: 832 head, tail = os.path.split(activity_path) 833 os.rename(activity_path, os.path.join(head, remote_base_dir)) 834 activity_path = os.path.join(head, remote_base_dir) 835 836 namelist = remote_zip.namelist() 837 infolist = remote_zip.infolist() 838 _logger.debug('base path %s' % activity_path) 839 840 # remove local files and directories that are not in new version 841 for root, dirs, files in os.walk(activity_path): 842 for fName in files: 843 local_path = os.path.join(root, fName) 844 archive_path = _relpath(local_path, activities_dir) 845 if archive_path not in namelist: 846 # file is not in new version, so delete local copy 847 _logger.debug("delete %s: removed in new version" % archive_path) 848 os.remove(local_path) 849 850 # we use [:] to make a copy of dirs to loop over. this is because 851 # we might modify it inside the loop, so we must use a copy. 852 for dName in dirs[:]: 853 local_path = os.path.join(root, dName) 854 archive_path = _relpath(local_path, activities_dir) + "/" 855 if archive_path not in namelist: 856 # directory no longer exists in new version, so wipe it out 857 _logger.debug("delete %s: removed in new version" % archive_path) 858 shutil.rmtree(local_path, ignore_errors=True) 859 # don't recurse into this directory, because it's gone 860 dirs.remove(dName) 861 862 # iterate over every file in the new version, downloading new files 863 # and updating ons that have changed 864 cnt = 0 865 num_ents = float(len(infolist)) 866 for ent in infolist: 867 cnt = cnt + 1 868 progress_cb(cnt / num_ents, row) 869 870 # don't extract to arbitrary locations 871 if os.path.isabs(ent.filename): 872 _logger.error("not extracting file with absolute path: %s" % ent.filename) 873 continue 874 filename = os.path.normpath(ent.filename) 875 if filename.startswith("../"): 876 _logger.error("not extracting file outside of activities dir: %s" % ent.filename) 877 continue 878 879 local_path = os.path.join(activities_dir, filename) 880 881 # create directories 882 if ent.filename.endswith("/"): 883 if not os.path.exists(local_path): 884 _logger.debug("create new directory %s" % ent.filename) 885 os.makedirs(local_path) 886 continue 887 888 if os.path.exists(local_path) and os.path.getsize(local_path) == ent.file_size and _file_crc32(local_path) == ent.CRC: 889 continue 890 891 _logger.debug("new or changed file: %s" % ent.filename) 892 _extract_from_zip(remote_zip, ent, activities_dir) 893 894 # activity.info is a special case. We do not want to lose it at any 895 # cost, so update it atomically. 896 local_act_info = os.path.join(activity_path, "activity", "activity.info") 897 fd = open(local_act_info + ".new", 'wb') 898 fd.write(remote_zip.read(os.path.join(remote_base_dir, "activity", "activity.info"))) 899 fd.close() 900 os.rename(local_act_info + ".new", local_act_info) 901 902 return old_path, new_path 703 903 704 904 ######################################################################### 705 905 # Control panel keys for command-line and GUI use. … … def set_install_update(which): 806 1006 loop = gobject.MainLoop() 807 1007 from sugar.activity.registry import get_registry 808 1008 registry = get_registry() 1009 1010 def updatehook(n, row): 1011 _print_status(n, _('Updating %s...') % row[DESCRIPTION_BIG]) 1012 for row, old_path, new_path in ul.do_selected_updates(updatehook): 1013 if old_path is None: continue # cancelled or network error 1014 fav = registry.get_activity(row[ACTIVITY_ID]).favorite 1015 registry.remove_bundle(old_path) 1016 registry.add_bundle(new_path) 1017 bundle = ActivityBundle(new_path) 1018 registry.set_activity_favorite(bundle.get_bundle_id(), bundle.get_activity_version(), fav) 1019 809 1020 def reporthook(n, row): 810 1021 _print_status(n, _('Downloading %s...') % row[DESCRIPTION_BIG]) 811 for row, f in ul.download_ selected_updates(reporthook):1022 for row, f in ul.download_new_activities(reporthook): 812 1023 if f is None: continue # cancelled or network error 813 1024 try: 814 1025 _print_status(None, _('Examining %s...') % row[DESCRIPTION_BIG]) -
view/updater.py
old new class ActivityUpdater(SectionView): 596 596 else: 597 597 progress_cb((n+(counts[0]/counts[1]))/2, extra, icon) 598 598 counts[2] = n # last fraction. 599 def q(n, row):599 def progress_new(n, row): 600 600 p(n, _('Downloading %s...') % row[model.DESCRIPTION_BIG], 601 601 row[model.ACTIVITY_ICON]) 602 for row, f in self.activity_list.download_selected_updates(q): 602 def progress_updates(n, row): 603 progress_cb((n+(counts[0]/counts[1])), 604 _('Updating %s...') % row[model.DESCRIPTION_BIG], 605 row[model.ACTIVITY_ICON]) 606 counts[2] = n # last fraction. 607 608 for row, old_path, new_path in self.activity_list.do_selected_updates(progress_updates): 609 if old_path is None: continue # cancelled or network error. 610 fav = registry.get_activity(row[model.ACTIVITY_ID]).favorite 611 registry.remove_bundle(old_path) 612 registry.add_bundle(new_path) 613 bundle = ActivityBundle(new_path) 614 registry.set_activity_favorite(bundle.get_bundle_id(), bundle.get_activity_version(), fav) 615 counts[0]+=1 616 617 for row, f in self.activity_list.download_new_activities(progress_new): 603 618 if f is None: continue # cancelled or network error. 604 619 try: 605 620 p(counts[2], _('Examining %s...')%row[model.DESCRIPTION_BIG],