Ticket #1636: tmpSVjeVO

diff --git a/src/jarabe/journal/listview.py b/src/jarabe/journal/listview.py
index 6556b08..9b287c4 100644
--- a/src/jarabe/journal/listview.py
+++ b/src/jarabe/journal/listview.py
@@ -123,6 +123,15 @@ class BaseListView(gtk.HBox):
                 datastore.connect_to_signal('Deleted',
                                             self.__datastore_deleted_cb)
 
+        model.created.connect(self.__model_updated_cb)
+        model.updated.connect(self.__model_updated_cb)
+        model.deleted.connect(self.__model_updated_cb)
+
+    def __model_updated_cb(self, sender, **kwargs):
+        if 'object_id' in kwargs and kwargs['object_id'].startswith('/'):
+            # otherwise rely on ds' Deleted event for ds objects
+            self._set_dirty()
+
     def __destroy_cb(self, widget):
         self._datastore_created_handler.remove()
         self._datastore_updated_handler.remove()
diff --git a/src/jarabe/journal/model.py b/src/jarabe/journal/model.py
index 9521456..b73e957 100644
--- a/src/jarabe/journal/model.py
+++ b/src/jarabe/journal/model.py
@@ -19,6 +19,7 @@ import os
 from datetime import datetime
 import time
 import shutil
+import tempfile
 from stat import S_IFMT, S_IFDIR, S_IFREG
 import traceback
 import re
@@ -27,6 +28,8 @@ import gobject
 import dbus
 import gconf
 import gio
+import json
+import base64
 
 from sugar import dispatch
 from sugar import mime
@@ -308,8 +311,10 @@ class InplaceResultSet(BaseResultSet):
         files = self._file_list[offset:offset + limit]
 
         entries = []
-        for file_path, stat, mtime_ in files:
-            metadata = _get_file_metadata(file_path, stat)
+        for file_path, stat, mtime_, metadata in files:
+            if metadata == False:
+                # fallback - the find should fetch md
+                metadata = _get_file_metadata(file_path, stat)
             metadata['mountpoint'] = self._mount_point
             entries.append(metadata)
 
@@ -334,9 +339,18 @@ class InplaceResultSet(BaseResultSet):
                 elif S_IFMT(stat.st_mode) == S_IFREG:
                     add_to_list = True
 
-                    if self._regex is not None and \
-                            not self._regex.match(full_path):
+                    md = _get_file_metadata_from_json(dir_path, entry,
+                                                       preview=False)
+                    if self._regex is not None:
                         add_to_list = False
+                        if self._regex.match(full_path):
+                            add_to_list = True
+                        elif md:
+                            # match any of the text md fields
+                            for f in ['fulltext', 'title', 'description', 'tags']:
+                                if f in md and self._regex.match(md[f]):
+                                    add_to_list = True
+                                    break
 
                     if None not in [self._date_start, self._date_end] and \
                             (stat.st_mtime < self._date_start or
@@ -349,7 +363,7 @@ class InplaceResultSet(BaseResultSet):
                             add_to_list = False
 
                     if add_to_list:
-                        file_info = (full_path, stat, int(stat.st_mtime))
+                        file_info = (full_path, stat, int(stat.st_mtime), md)
                         self._file_list.append(file_info)
 
                     self.progress.send(self)
@@ -364,6 +378,15 @@ class InplaceResultSet(BaseResultSet):
             self._pending_directories -= 1
 
 def _get_file_metadata(path, stat):
+    # will sometimes be called for
+    # files that do have a metatada file
+    fname = os.path.basename(path)
+    dir_path = os.path.dirname(path)
+    md = _get_file_metadata_from_json(dir_path, fname, preview=True)
+    if md:
+        return md
+
+    # make up something for files w/o metadata
     client = gconf.client_get_default()
     return {'uid': path,
             'title': os.path.basename(path),
@@ -374,6 +397,30 @@ def _get_file_metadata(path, stat):
             'icon-color': client.get_string('/desktop/sugar/user/color'),
             'description': path}
 
+def _get_file_metadata_from_json(dir_path, fname, preview=False):
+    md = None
+    mdpath = os.path.join(dir_path,
+                          '.'+fname+'.metadata')
+    md = False
+    if os.path.exists(mdpath):
+        try:
+            md = json.load(open(mdpath))
+            md['uid'] = os.path.join(dir_path, fname)
+        except:
+            pass
+    if preview:
+        prpath = os.path.join(dir_path,
+                              '.'+fname+'.preview')
+        try:
+            preview = base64.b64encode(open(prpath).read())
+            md['preview'] = preview
+        except:
+            pass
+    else:
+        if md and 'preview' in md:
+            del(md['preview'])
+    return md
+
 _datastore = None
 def _get_datastore():
     global _datastore
@@ -460,6 +507,18 @@ def delete(object_id):
     """
     if os.path.exists(object_id):
         os.unlink(object_id)
+        dir_path = os.path.dirname(object_id)
+        fname = os.path.basename(object_id)
+        old_files = [ os.path.join(dir_path,
+                                   '.'+fname+'.metadata'),
+                      os.path.join(dir_path,
+                                   '.'+fname+'.preview') ]
+        for ofile in old_files:
+            if os.path.exists(ofile):
+                try:
+                    os.unlink(ofile)
+                except:
+                    pass
         deleted.send(None, object_id=object_id)
     else:
         _get_datastore().delete(object_id)
@@ -480,6 +539,7 @@ def write(metadata, file_path='', update_mtime=True):
     """
     logging.debug('model.write %r %r %r' % (metadata.get('uid', ''), file_path,
                                             update_mtime))
+
     if update_mtime:
         metadata['mtime'] = datetime.now().isoformat()
         metadata['timestamp'] = int(time.time())
@@ -495,16 +555,87 @@ def write(metadata, file_path='', update_mtime=True):
                                                  file_path,
                                                  True)
     else:
-        if not os.path.exists(file_path):
+        if 'uid' in metadata and os.path.exists(metadata['uid']):
+            file_path = metadata['uid']
+
+        if not file_path or not os.path.exists(file_path):
             raise ValueError('Entries without a file cannot be copied to '
                              'removable devices')
 
+        # cases for file_name:
+        #  - metadata update -> keep same fname
+        #  - new file / copy from DS -> check for name collision
+        #  - renaming file on external storage -> check for collision
         file_name = _get_file_name(metadata['title'], metadata['mime_type'])
-        file_name = _get_unique_file_name(metadata['mountpoint'], file_name)
-
         destination_path = os.path.join(metadata['mountpoint'], file_name)
-        shutil.copy(file_path, destination_path)
+        if destination_path != file_path:
+            # new file (copied from DS) or rename
+            if os.path.exists(destination_path):
+                # diverting to a new filename to avoid collision
+                file_name = _get_unique_file_name(metadata['mountpoint'], file_name)
+                destination_path = os.path.join(metadata['mountpoint'], file_name)
+                # reflect this in title - to avoid misleading titles in ext media
+                cleanname, extension = os.path.splitext(file_name)
+                metadata['title'] = cleanname
+
+        ## Prepare and write the metadata out first
+        ## this is sane for new objects (copy from DS)
+        ## and renames. Additionally, it makes renames
+        ## failsafe.
+
+        # issue our metadata massage on a copy
+        md = metadata.copy()
+        del md['mountpoint']
+
+        # if we ever write to places other than the root
+        # keep in mind that the metadata file must be in the
+        # same directory as the data file.
+        md_path = os.path.join(metadata['mountpoint'],
+                               '.'+file_name+'.metadata')
+
+        if 'preview' in md:
+            preview = md['preview']
+            preview_fname =  '.'+file_name+'.preview'
+            preview_path = os.path.join(metadata['mountpoint'], preview_fname)
+            md['preview'] = preview_fname
+
+            # Write preview atomically
+            (fh, fn) = tempfile.mkstemp(dir=metadata['mountpoint'])
+            os.write(fh, preview)
+            os.close(fh)
+            os.rename(fn, preview_path)
+
+        # Write metadata atomically (on FSs that support it)
+        (fh, fn) = tempfile.mkstemp(dir=metadata['mountpoint'])
+        os.write(fh, json.dumps(md))
+        os.close(fh)
+        os.rename(fn, md_path)
+
+        # Now deal with the actual file
+        if os.path.dirname(destination_path) == os.path.dirname(file_path):
+            # same dir case - updated metadata, or rename
+            old_file_path = file_path
+            if old_file_path != destination_path:
+                # title change leads to rename on removable disk
+                os.rename(file_path, destination_path)
+                old_fname = os.path.basename(file_path)
+                old_files = [ os.path.join(metadata['mountpoint'],
+                                           '.'+old_fname+'.metadata'),
+                              os.path.join(metadata['mountpoint'],
+                                           '.'+old_fname+'.preview') ]
+                for ofile in old_files:
+                    if os.path.exists(ofile):
+                        try:
+                            os.unlink(ofile)
+                        except:
+                            pass
+        else:
+            # different dir - likely copy from DS
+            shutil.copy(file_path, destination_path)
+
+
         object_id = destination_path
+        metadata['uid'] = object_id
         created.send(None, object_id=object_id)
 
     return object_id
@@ -512,9 +643,11 @@ def write(metadata, file_path='', update_mtime=True):
 def _get_file_name(title, mime_type):
     file_name = title
 
-    extension = '.' + mime.get_primary_extension(mime_type)
-    if not file_name.endswith(extension):
-        file_name += extension
+    mime_extension = mime.get_primary_extension(mime_type)
+    if mime_extension:
+        extension = '.' + mime_extension
+        if not file_name.endswith(extension):
+            file_name += extension
 
     # Invalid characters in VFAT filenames. From
     # http://en.wikipedia.org/wiki/File_Allocation_Table
@@ -534,14 +667,14 @@ def _get_file_name(title, mime_type):
 def _get_unique_file_name(mount_point, file_name):
     if os.path.exists(os.path.join(mount_point, file_name)):
         i = 1
+        name, extension = os.path.splitext(file_name)
         while len(file_name) <= 255:
-            name, extension = os.path.splitext(file_name)
-            file_name = name + '_' + str(i) + extension
-            if not os.path.exists(os.path.join(mount_point, file_name)):
+            try_file_name = name + '_' + str(i) + extension
+            if not os.path.exists(os.path.join(mount_point, try_file_name)):
                 break
             i += 1
 
-    return file_name
+    return try_file_name
 
 created = dispatch.Signal()
 updated = dispatch.Signal()