Ticket #1499: activity-update-incremental-v2.patch
File activity-update-incremental-v2.patch, 14.0 KB (added by dsd, 15 years ago) |
---|
-
model/updater.py
old new import gtk 27 27 import gobject 28 28 29 29 import locale 30 import logging 30 31 import os 31 32 import os.path 32 33 import socket 34 import shutil 33 35 import sys 34 36 import traceback 35 37 import zipfile 38 import zlib 36 39 from HTMLParser import HTMLParseError 37 40 from urllib2 import HTTPError 38 41 … … import bitfrost.update.actutils as actut 44 47 import bitfrost.update.microformat as microformat 45 48 import bitfrost.util.urlrange as urlrange 46 49 50 _logger = logging.getLogger("updater") 51 47 52 # weak dependency on inhibit_suspend from olpc-update package 48 53 try: 49 54 from bitfrost.update import inhibit_suspend … … def _humanize_size(bytes): 69 74 # TRANSLATORS: download size of updates, e.g. "2.3 MB" 70 75 return locale.format(_("%.1f MB"), bytes / 1024 / 1024) 71 76 77 def _file_crc32(path): 78 fd = open(path, 'rb') 79 crc = 0 80 81 while True: 82 buffer = fd.read(8192) 83 if not buffer: 84 break 85 crc = zlib.crc32(buffer, crc) 86 87 fd.close() 88 return crc 89 90 # FIXME use os.path.relpath when Python2.6 is required 91 def _relpath(path, start): 92 start_list = os.path.abspath(start).split('/') 93 path_list = os.path.abspath(path).split('/') 94 i = len(os.path.commonprefix([start_list, path_list])) 95 rel_list = [".."] * (len(start_list)-i) + path_list[i:] 96 if not rel_list: 97 return start 98 return os.path.join(*rel_list) 99 100 def _extract_from_zip(zip, zinfo, base_dir): 101 fd = open(os.path.join(base_dir, zinfo.filename), 'wb') 102 103 # python-2.5.1 zipfile does not like read() of a 0 byte file 104 if zinfo.file_size > 0: 105 fd.write(zip.read(zinfo.filename)) 106 107 fd.close() 108 72 109 def _svg2pixbuf(icon_data): 73 110 """Convert the given `icon_data` SVG string to a `gtk.gdk.Pixbuf` 74 111 with maximum size 55x55.""" … … class UpdateList(gtk.ListStore): 489 526 self._cancel = True 490 527 491 528 def cancel_download(self): 492 """Asynchronously cancel a `download_selected_updates`operation."""529 """Asynchronously cancel an update operation.""" 493 530 self._cancel = True 494 531 495 532 def _sum_rows(self, row_func): … … class UpdateList(gtk.ListStore): 508 545 return self._sum_rows(lambda r: 1 if 509 546 r[UPDATE_EXISTS] and r[UPDATE_SELECTED] else 0) 510 547 def updates_size(self): 511 """Returns the size (in bytes) of the selected updates available. 548 """Returns the size (in bytes) of the selected updates and new 549 activities available. 512 550 513 551 Updated by `refresh`.""" 514 552 return self._sum_rows(lambda r: r[UPDATE_SIZE] if 515 553 r[UPDATE_EXISTS] and r[UPDATE_SELECTED] else 0) 554 def updates_size_new_only(self): 555 """Returns the size (in bytes) of the selected new activities. 556 557 Updated by `refresh`.""" 558 return self._sum_rows(lambda r: r[UPDATE_SIZE] if 559 r[UPDATE_EXISTS] and r[UPDATE_SELECTED] and 560 r[ACTIVITY_BUNDLE] is None else 0) 561 def updates_size_updates_only(self): 562 """Returns the size (in bytes) of the selected updates available. 563 564 Updated by `refresh`.""" 565 return self._sum_rows(lambda r: r[UPDATE_SIZE] if 566 r[UPDATE_EXISTS] and r[UPDATE_SELECTED] and 567 r[ACTIVITY_BUNDLE] is not None else 0) 516 568 def unselect_all(self): 517 569 """Unselect all available updates.""" 518 570 for row in self: … … class UpdateList(gtk.ListStore): 657 709 if row[IS_HEADER] and row[UPDATE_URL] is not None: 658 710 print >>f, row[UPDATE_URL] 659 711 660 def download_ selected_updates(self, progress_cb=(lambda n, row: None),661 712 def download_new_activities(self, progress_cb=(lambda n, row: None), 713 dir=None): 662 714 """Return a generator giving (row, local filename) pairs for 663 715 each selected update. Caller is responsible for unlinking 664 716 these files when they are through. The `progress_cb` gets a … … class UpdateList(gtk.ListStore): 674 726 if dir is None: dir=actinfo.USER_ACTIVITY_DIR 675 727 if not os.path.isdir(dir): os.makedirs(dir) 676 728 self._cancel = False 677 sizes = [ 0, self.updates_size () ]729 sizes = [ 0, self.updates_size_new_only() ] 678 730 for row in self: 679 731 if self._cancel: return # bail. 680 732 if row[IS_HEADER]: continue 681 733 if not (row[UPDATE_EXISTS] and row[UPDATE_SELECTED]): continue 734 735 # dont handle updates here 736 if row[ACTIVITY_BUNDLE] is not None: continue 737 682 738 def report(cursize, totalsize_): 683 739 if totalsize_ is None: 684 740 progress_cb(None, row) … … class UpdateList(gtk.ListStore): 700 756 sizes[0] += row[UPDATE_SIZE] 701 757 progress_cb(sizes[0] / sizes[1], row) 702 758 759 def do_selected_updates(self, progress_cb): 760 """ 761 Update activities by updating their files incrementally, only 762 downloading the files that have changed. 763 """ 764 765 self._cancel = False 766 sizes = [ 0, self.updates_size_updates_only() ] 767 768 def _progress_cb(n, row): 769 progress_cb((sizes[0] + (n*row[UPDATE_SIZE])) / sizes[1], row) 770 771 for row in self: 772 if self._cancel: return # bail. 773 if row[IS_HEADER]: continue 774 if not (row[UPDATE_EXISTS] and row[UPDATE_SELECTED]): continue 775 if row[ACTIVITY_BUNDLE] is None: continue # updates only 776 777 _logger.debug("incremental update %s" % row[ACTIVITY_ID]) 778 old_path, new_path = self.do_inplace_update(row, _progress_cb) 779 sizes[0] += row[UPDATE_SIZE] 780 progress_cb(sizes[0] / sizes[1], row) 781 yield row, old_path, new_path 782 783 def do_inplace_update(self, row, progress_cb): 784 """ 785 Incrementally update an activity according to the following algorithm: 786 787 1. If the base activity directory has changed, rename the local version 788 2. Delete any local files and directories that are not present in the 789 upstream version 790 3. Download files from the upstream version that are not present 791 locally, or if the local versions differ. 792 4. Atomically update activity.info 793 794 Deletion is done first, because the disk space freed up by deletion 795 may be required for the downloading of files in step 3. 796 797 activity.info update is atomic so that you are never left in the 798 situation where you cannot delete the activity from sugar (or re-run 799 the updater). 800 801 This operation cannot be cancelled. 802 """ 803 try: 804 remote_zip = zipfile.ZipFile(urlrange.urlopen(row[UPDATE_URL], use_cache=False, timeout=30), 'r') 805 except (HTMLParseError, IOError, socket.error): 806 return None, None # network error 807 808 # e.g. /home/olpc/Activities/Foo.activity 809 activity_path = row[ACTIVITY_BUNDLE].get_path().rstrip('/') 810 _logger.debug("activity_path %s" %activity_path) 811 812 # e.g. /home/olpc/Activities 813 activities_dir = os.path.dirname(activity_path) 814 _logger.debug("activities_dir %s" % activities_dir) 815 816 # e.g. Foo.activity (name of activity directory on local disk) 817 local_base_dir = activity_path.split('/')[-1] 818 _logger.debug("local base dir %s" % local_base_dir) 819 820 # e.g. Foo.activity (name of activity directory in remote zipfile) 821 remote_base_dir = actutils.bundle_base_from_zipfile(remote_zip) 822 _logger.debug("remote base dir %s" % remote_base_dir) 823 824 # e.g. Foo.activity/activity/activity.info 825 remote_act_info = os.path.join(remote_base_dir, "activity", "activity.info") 826 827 old_path = activity_path 828 new_path = os.path.join(activities_dir, remote_base_dir) 829 830 # last opportunity to safely cancel 831 if self._cancel: 832 return None, None 833 834 if local_base_dir != remote_base_dir: 835 head, tail = os.path.split(activity_path) 836 os.rename(activity_path, os.path.join(head, remote_base_dir)) 837 activity_path = os.path.join(head, remote_base_dir) 838 839 namelist = remote_zip.namelist() 840 infolist = remote_zip.infolist() 841 _logger.debug('base path %s' % activity_path) 842 843 # remove local files and directories that are not in new version 844 for root, dirs, files in os.walk(activity_path): 845 for fName in files: 846 local_path = os.path.join(root, fName) 847 archive_path = _relpath(local_path, activities_dir) 848 if archive_path not in namelist: 849 # file is not in new version, so delete local copy 850 _logger.debug("delete %s: removed in new version" % archive_path) 851 os.remove(local_path) 852 853 # we use [:] to make a copy of dirs to loop over. this is because 854 # we might modify it inside the loop, so we must use a copy. 855 for dName in dirs[:]: 856 local_path = os.path.join(root, dName) 857 archive_path = _relpath(local_path, activities_dir) + "/" 858 if archive_path not in namelist: 859 # directory no longer exists in new version, so wipe it out 860 _logger.debug("delete %s: removed in new version" % archive_path) 861 shutil.rmtree(local_path, ignore_errors=True) 862 # don't recurse into this directory, because it's gone 863 dirs.remove(dName) 864 865 # iterate over every file in the new version, downloading new files 866 # and updating ons that have changed 867 cnt = 0 868 num_ents = float(len(infolist)) 869 for ent in infolist: 870 cnt = cnt + 1 871 progress_cb(cnt / num_ents, row) 872 873 # don't extract to arbitrary locations 874 if os.path.isabs(ent.filename): 875 _logger.error("not extracting file with absolute path: %s" % ent.filename) 876 continue 877 filename = os.path.normpath(ent.filename) 878 if filename.startswith("../"): 879 _logger.error("not extracting file outside of activities dir: %s" % ent.filename) 880 continue 881 882 if ent.filename == remote_act_info: 883 _logger.debug("skip activity.info til later") 884 continue 885 886 local_path = os.path.join(activities_dir, filename) 887 888 # create directories 889 if ent.filename.endswith("/"): 890 if not os.path.exists(local_path): 891 _logger.debug("create new directory %s" % ent.filename) 892 os.makedirs(local_path) 893 continue 894 895 if os.path.exists(local_path) and os.path.getsize(local_path) == ent.file_size and _file_crc32(local_path) == ent.CRC: 896 continue 897 898 _logger.debug("new or changed file: %s" % ent.filename) 899 _extract_from_zip(remote_zip, ent, activities_dir) 900 901 # activity.info is a special case. We do not want to lose it at any 902 # cost, so update it atomically. 903 if remote_act_info in namelist: 904 local_act_info = os.path.join(activity_path, "activity", "activity.info") 905 fd = open(local_act_info + ".new", 'wb') 906 fd.write(remote_zip.read(remote_act_info)) 907 fd.close() 908 os.rename(local_act_info + ".new", local_act_info) 909 910 return old_path, new_path 703 911 704 912 ######################################################################### 705 913 # Control panel keys for command-line and GUI use. … … def set_install_update(which): 806 1014 loop = gobject.MainLoop() 807 1015 from sugar.activity.registry import get_registry 808 1016 registry = get_registry() 1017 1018 def updatehook(n, row): 1019 _print_status(n, _('Updating %s...') % row[DESCRIPTION_BIG]) 1020 for row, old_path, new_path in ul.do_selected_updates(updatehook): 1021 if old_path is None: continue # cancelled or network error 1022 b = actutils.BundleHelper(new_path) 1023 b.handle_inplace_upgrade(registry, old_path) 1024 809 1025 def reporthook(n, row): 810 1026 _print_status(n, _('Downloading %s...') % row[DESCRIPTION_BIG]) 811 for row, f in ul.download_ selected_updates(reporthook):1027 for row, f in ul.download_new_activities(reporthook): 812 1028 if f is None: continue # cancelled or network error 813 1029 try: 814 1030 _print_status(None, _('Examining %s...') % row[DESCRIPTION_BIG]) -
view/updater.py
old new class ActivityUpdater(SectionView): 596 596 else: 597 597 progress_cb((n+(counts[0]/counts[1]))/2, extra, icon) 598 598 counts[2] = n # last fraction. 599 def q(n, row):599 def progress_new(n, row): 600 600 p(n, _('Downloading %s...') % row[model.DESCRIPTION_BIG], 601 601 row[model.ACTIVITY_ICON]) 602 for row, f in self.activity_list.download_selected_updates(q): 602 def progress_updates(n, row): 603 progress_cb((n+(counts[0]/counts[1])), 604 _('Updating %s...') % row[model.DESCRIPTION_BIG], 605 row[model.ACTIVITY_ICON]) 606 counts[2] = n # last fraction. 607 608 for row, old_path, new_path in self.activity_list.do_selected_updates(progress_updates): 609 if old_path is None: continue # cancelled or network error. 610 b = actutils.BundleHelper(new_path) 611 b.handle_inplace_upgrade(registry, old_path) 612 counts[0]+=1 613 614 for row, f in self.activity_list.download_new_activities(progress_new): 603 615 if f is None: continue # cancelled or network error. 604 616 try: 605 617 p(counts[2], _('Examining %s...')%row[model.DESCRIPTION_BIG],