Ticket #1499: activity-update-incremental.patch

model/updater.py

@@ -27 +27 @@
 import gobject
 
 import locale
+import logging
 import os
 import os.path
 import socket
+import shutil
 import sys
 import traceback
 import zipfile
+import zlib
 from HTMLParser import HTMLParseError
 from urllib2 import HTTPError
 
@@ -44 +47 @@
 import bitfrost.update.microformat as microformat
 import bitfrost.util.urlrange as urlrange
 
+_logger = logging.getLogger("updater")
+
 # weak dependency on inhibit_suspend from olpc-update package
 try:
     from bitfrost.update import inhibit_suspend
@@ -69 +74 @@
         # TRANSLATORS: download size of updates, e.g. "2.3 MB"
         return locale.format(_("%.1f MB"), bytes / 1024 / 1024)
 
+def _file_crc32(path):
+    fd = open(path, 'rb')
+    crc = 0
+
+    while True:
+        buffer = fd.read(8192)
+        if not buffer:
+            break
+        crc = zlib.crc32(buffer, crc)
+
+    fd.close()
+    return crc
+
+# FIXME use os.path.relpath when Python2.6 is required
+def _relpath(path, start):
+    start_list = os.path.abspath(start).split('/')
+    path_list = os.path.abspath(path).split('/')
+    i = len(os.path.commonprefix([start_list, path_list]))
+    rel_list = [".."] * (len(start_list)-i) + path_list[i:]
+    if not rel_list:
+        return start
+    return os.path.join(*rel_list)
+
+def _extract_from_zip(zip, zinfo, base_dir):
+    fd = open(os.path.join(base_dir, zinfo.filename), 'wb')
+
+    # python-2.5.1 zipfile does not like read() of a 0 byte file
+    if zinfo.file_size > 0:
+        fd.write(zip.read(zinfo.filename))
+
+    fd.close()
+
 def _svg2pixbuf(icon_data):
     """Convert the given `icon_data` SVG string to a `gtk.gdk.Pixbuf`
     with maximum size 55x55."""
@@ -489 +526 @@
         self._cancel = True
 
     def cancel_download(self):
-        """Asynchronously cancel a `download_selected_updates` operation."""
+        """Asynchronously cancel an update operation."""
         self._cancel = True
 
     def _sum_rows(self, row_func):
@@ -508 +545 @@
         return self._sum_rows(lambda r: 1 if
                               r[UPDATE_EXISTS] and r[UPDATE_SELECTED] else 0)
     def updates_size(self):
-        """Returns the size (in bytes) of the selected updates available.
+        """Returns the size (in bytes) of the selected updates and new
+        activities available.
 
         Updated by `refresh`."""
         return self._sum_rows(lambda r: r[UPDATE_SIZE] if
                               r[UPDATE_EXISTS] and r[UPDATE_SELECTED] else 0)
+    def updates_size_new_only(self):
+        """Returns the size (in bytes) of the selected new activities.
+
+        Updated by `refresh`."""
+        return self._sum_rows(lambda r: r[UPDATE_SIZE] if
+                              r[UPDATE_EXISTS] and r[UPDATE_SELECTED] and
+                              r[ACTIVITY_BUNDLE] is None else 0)
+    def updates_size_updates_only(self):
+        """Returns the size (in bytes) of the selected updates available.
+
+        Updated by `refresh`."""
+        return self._sum_rows(lambda r: r[UPDATE_SIZE] if
+                              r[UPDATE_EXISTS] and r[UPDATE_SELECTED] and
+                              r[ACTIVITY_BUNDLE] is not None else 0)
     def unselect_all(self):
         """Unselect all available updates."""
         for row in self:
@@ -657 +709 @@
                 if row[IS_HEADER] and row[UPDATE_URL] is not None:
                     print >>f, row[UPDATE_URL]
 
-    def download_selected_updates(self, progress_cb=(lambda n, row: None),
-                                  dir=None):
+    def download_new_activities(self, progress_cb=(lambda n, row: None),
+                                dir=None):
         """Return a generator giving (row, local filename) pairs for
         each selected update.  Caller is responsible for unlinking
         these files when they are through.  The `progress_cb` gets a
@@ -674 +726 @@
         if dir is None: dir=actinfo.USER_ACTIVITY_DIR
         if not os.path.isdir(dir): os.makedirs(dir)
         self._cancel = False
-        sizes = [ 0, self.updates_size() ]
+        sizes = [ 0, self.updates_size_new_only() ]
         for row in self:
             if self._cancel: return # bail.
             if row[IS_HEADER]: continue
             if not (row[UPDATE_EXISTS] and row[UPDATE_SELECTED]): continue
+
+            # dont handle updates here
+            if row[ACTIVITY_BUNDLE] is not None: continue
+
             def report(cursize, totalsize_):
                 if totalsize_ is None:
                     progress_cb(None, row)
@@ -700 +756 @@
             sizes[0] += row[UPDATE_SIZE]
             progress_cb(sizes[0] / sizes[1], row)
 
+    def do_selected_updates(self, progress_cb):
+        """
+        Update activities by updating their files incrementally, only
+        downloading the files that have changed.
+        """
+
+        self._cancel = False
+        sizes = [ 0, self.updates_size_updates_only() ]
+
+        def _progress_cb(n, row):
+            progress_cb((sizes[0] + (n*row[UPDATE_SIZE])) / sizes[1], row)
+
+        for row in self:
+            if self._cancel: return # bail.
+            if row[IS_HEADER]: continue
+            if not (row[UPDATE_EXISTS] and row[UPDATE_SELECTED]): continue
+            if row[ACTIVITY_BUNDLE] is None: continue # updates only
+
+            _logger.debug("incremental update %s" % row[ACTIVITY_ID])
+            old_path, new_path = self.do_inplace_update(row, _progress_cb)
+            sizes[0] += row[UPDATE_SIZE]
+            progress_cb(sizes[0] / sizes[1], row)
+            yield row, old_path, new_path
+
+    def do_inplace_update(self, row, progress_cb):
+        """
+        Incrementally update an activity according to the following algorithm:
+
+        1. If the base activity directory has changed, rename the local version
+        2. Delete any local files and directories that are not present in the
+           upstream version
+        3. Download files from the upstream version that are not present
+           locally, or if the local versions differ.
+        4. Atomically update activity.info
+
+        Deletion is done first, because the disk space freed up by deletion
+        may be required for the downloading of files in step 3.
+        
+        activity.info update is atomic so that you are never left in the
+        situation where you cannot delete the activity from sugar (or re-run
+        the updater).
+
+        This operation cannot be cancelled.
+        """
+        try:
+            remote_zip = zipfile.ZipFile(urlrange.urlopen(row[UPDATE_URL], use_cache=False, timeout=30), 'r')
+        except (HTMLParseError, IOError, socket.error):
+            return None, None # network error
+
+        # e.g. /home/olpc/Activities/Foo.activity
+        activity_path = row[ACTIVITY_BUNDLE].get_path().rstrip('/')
+        _logger.debug("activity_path %s" %activity_path)
+
+        # e.g. /home/olpc/Activities
+        activities_dir = os.path.dirname(activity_path)
+        _logger.debug("activities_dir %s" % activities_dir)
+
+        # e.g. Foo.activity (name of activity directory on local disk)
+        local_base_dir = activity_path.split('/')[-1]
+        _logger.debug("local base dir %s" % local_base_dir)
+
+        # e.g. Foo.activity (name of activity directory in remote zipfile)
+        remote_base_dir = actutils.bundle_base_from_zipfile(remote_zip)
+        _logger.debug("remote base dir %s" % remote_base_dir)
+
+        old_path = activity_path
+        new_path = os.path.join(activities_dir, remote_base_dir)
+
+        # last opportunity to safely cancel
+        if self._cancel:
+            return None, None
+
+        if local_base_dir != remote_base_dir:
+            head, tail = os.path.split(activity_path)
+            os.rename(activity_path, os.path.join(head, remote_base_dir))
+            activity_path = os.path.join(head, remote_base_dir)
+
+        namelist = remote_zip.namelist()
+        infolist = remote_zip.infolist()
+        _logger.debug('base path %s' % activity_path)
+
+        # remove local files and directories that are not in new version
+        for root, dirs, files in os.walk(activity_path):
+            for fName in files:
+                local_path = os.path.join(root, fName)
+                archive_path = _relpath(local_path, activities_dir)
+                if archive_path not in namelist:
+                    # file is not in new version, so delete local copy
+                    _logger.debug("delete %s: removed in new version" % archive_path)
+                    os.remove(local_path)
+
+            # we use [:] to make a copy of dirs to loop over. this is because
+            # we might modify it inside the loop, so we must use a copy.
+            for dName in dirs[:]:
+                local_path = os.path.join(root, dName)
+                archive_path = _relpath(local_path, activities_dir) + "/"
+                if archive_path not in namelist:
+                    # directory no longer exists in new version, so wipe it out
+                    _logger.debug("delete %s: removed in new version" % archive_path)
+                    shutil.rmtree(local_path, ignore_errors=True)
+                    # don't recurse into this directory, because it's gone
+                    dirs.remove(dName)
+
+        # iterate over every file in the new version, downloading new files
+        # and updating ons that have changed
+        cnt = 0
+        num_ents = float(len(infolist))
+        for ent in infolist:
+            cnt = cnt + 1
+            progress_cb(cnt / num_ents, row)
+
+            # don't extract to arbitrary locations
+            if os.path.isabs(ent.filename):
+                _logger.error("not extracting file with absolute path: %s" % ent.filename)
+                continue
+            filename = os.path.normpath(ent.filename)
+            if filename.startswith("../"):
+                _logger.error("not extracting file outside of activities dir: %s" % ent.filename)
+                continue
+
+            local_path = os.path.join(activities_dir, filename)
+
+            # create directories
+            if ent.filename.endswith("/"):
+                if not os.path.exists(local_path):
+                    _logger.debug("create new directory %s" % ent.filename)
+                    os.makedirs(local_path)
+                continue
+
+            if os.path.exists(local_path) and os.path.getsize(local_path) == ent.file_size and _file_crc32(local_path) == ent.CRC:
+                continue
+
+            _logger.debug("new or changed file: %s" % ent.filename)
+            _extract_from_zip(remote_zip, ent, activities_dir)
+
+        # activity.info is a special case. We do not want to lose it at any
+        # cost, so update it atomically.
+        local_act_info = os.path.join(activity_path, "activity", "activity.info")
+        fd = open(local_act_info + ".new", 'wb')
+        fd.write(remote_zip.read(os.path.join(remote_base_dir, "activity", "activity.info")))
+        fd.close()
+        os.rename(local_act_info + ".new", local_act_info)
+
+        return old_path, new_path
 
 #########################################################################
 # Control panel keys for command-line and GUI use.
@@ -806 +1006 @@
     loop = gobject.MainLoop()
     from sugar.activity.registry import get_registry
     registry = get_registry()
+
+    def updatehook(n, row):
+        _print_status(n, _('Updating %s...') % row[DESCRIPTION_BIG])
+    for row, old_path, new_path in ul.do_selected_updates(updatehook):
+        if old_path is None: continue # cancelled or network error
+        fav = registry.get_activity(row[ACTIVITY_ID]).favorite
+        registry.remove_bundle(old_path)
+        registry.add_bundle(new_path)
+        bundle = ActivityBundle(new_path)
+        registry.set_activity_favorite(bundle.get_bundle_id(), bundle.get_activity_version(), fav)
+
     def reporthook(n, row):
         _print_status(n, _('Downloading %s...') % row[DESCRIPTION_BIG])
-    for row, f in ul.download_selected_updates(reporthook):
+    for row, f in ul.download_new_activities(reporthook):
         if f is None: continue # cancelled or network error
         try:
             _print_status(None, _('Examining %s...') % row[DESCRIPTION_BIG])

view/updater.py

@@ -596 +596 @@
                 else:
                     progress_cb((n+(counts[0]/counts[1]))/2, extra, icon)
                 counts[2] = n # last fraction.
-            def q(n, row):
+            def progress_new(n, row):
                 p(n, _('Downloading %s...') % row[model.DESCRIPTION_BIG],
                   row[model.ACTIVITY_ICON])
-            for row, f in self.activity_list.download_selected_updates(q):
+            def progress_updates(n, row):
+                progress_cb((n+(counts[0]/counts[1])),
+                            _('Updating %s...') % row[model.DESCRIPTION_BIG],
+                            row[model.ACTIVITY_ICON])
+                counts[2] = n # last fraction.
+
+            for row, old_path, new_path in self.activity_list.do_selected_updates(progress_updates):
+                if old_path is None: continue # cancelled or network error.
+                fav = registry.get_activity(row[model.ACTIVITY_ID]).favorite
+                registry.remove_bundle(old_path)
+                registry.add_bundle(new_path)
+                bundle = ActivityBundle(new_path)
+                registry.set_activity_favorite(bundle.get_bundle_id(), bundle.get_activity_version(), fav)
+                counts[0]+=1
+
+            for row, f in self.activity_list.download_new_activities(progress_new):
                 if f is None: continue # cancelled or network error.
                 try:
                     p(counts[2], _('Examining %s...')%row[model.DESCRIPTION_BIG],