Ticket #1124: ds-backup.py

File ds-backup.py, 5.4 KB (added by hamiltonchua, 12 years ago)
Line 
1#!/usr/bin/python
2# -*- coding: utf-8 -*-
3# Copyright (C) 2007 Ivan Krstić
4# Copyright (C) 2007 Tomeu Vizoso
5# Copyright (C) 2007 One Laptop per Child
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of version 2 of the GNU General Public License (and
9# no other version) as published by the Free Software Foundation.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19
20import os
21import sha
22import urllib2
23from urllib2 import URLError, HTTPError
24import os.path
25import tempfile
26import time
27import glob
28import popen2
29import re
30import gconf
31
32from sugar import env
33from sugar import profile
34
35class BackupError(Exception): pass
36class ProtocolVersionError(BackupError): pass
37class RefusedByServerError(BackupError): pass
38class ServerTooBusyError(BackupError): pass
39class TransferError(BackupError): pass
40class NoPriorBackups(BackupError): pass
41class BulkRestoreUnavailable(BackupError): pass
42
43def check_server_available(server, xo_serial):
44
45    try:
46        ret = urllib2.urlopen(server + '/available/%s' % xo_serial).read()
47        return 200
48    except HTTPError, e:
49        # server is there, did not fullfull req
50        #  expect 404, 403, 503 as e[1]
51        return e.code
52    except URLError, e:
53        # log it?
54        # print e.reason
55        return -1
56
57def rsync_to_xs(from_path, to_path, keyfile, user):
58
59    # add a trailing slash to ensure
60    # that we don't generate a subdir
61    # at the remote end. rsync oddities...
62    if not re.compile('/$').search(from_path):
63        from_path = from_path + '/'
64
65    ssh = '/usr/bin/ssh -F /dev/null -o "PasswordAuthentication no" -o "StrictHostKeyChecking no" -i "%s" -l "%s"' \
66        % (keyfile, user)
67    rsync = "/usr/bin/rsync -a -z -rlt --partial --delete --timeout=160 -e '%s' '%s' '%s' " % \
68            (ssh, from_path, to_path)
69    print rsync
70    rsync_p = popen2.Popen3(rsync, True)
71
72    # here we could track progress with a
73    # for line in pipe:
74    # (an earlier version had it)
75
76    # wait() returns a DWORD, we want the lower
77    # byte of that.
78    rsync_exit = os.WEXITSTATUS(rsync_p.wait())
79    if rsync_exit != 0:
80        # TODO: retry a couple of times
81        # if rsync_exit is 30 (Timeout in data send/receive)
82        raise TransferError('rsync error code %s, message:'
83                            % rsync_exit, rsync_p.childerr.read())
84
85    # Transfer an empty file marking completion
86    # so the XS can see we are done.
87    # Note: the dest dir on the XS is watched via
88    # inotify - so we avoid creating tempfiles there.
89    tmpfile = tempfile.mkstemp()
90    rsync = ("/usr/bin/rsync -a -z -rlt --timeout 10 -T /tmp -e '%s' '%s' '%s' "
91             % (ssh, tmpfile[1], BACKUP_SERVER+':/var/lib/ds-backup/completion/'+user))
92    rsync_p = popen2.Popen3(rsync, True)
93    rsync_exit = os.WEXITSTATUS(rsync_p.wait())
94    if rsync_exit != 0:
95        # TODO: retry a couple ofd times
96        # if rsync_exit is 30 (Timeout in data send/receive)
97        raise TransferError('rsync error code %s, message:'
98                            % rsync_exit, rsync_p.childerr.read())
99
100def have_ofw_tree():
101    return os.path.exists('/ofw')
102
103def read_ofw(path):
104    path = os.path.join('/ofw', path)
105    if not os.path.exists(path):
106        return None
107    fh = open(path, 'r')
108    data = fh.read().rstrip('\0\n')
109    fh.close()
110    return data
111
112# SoaS : def to retreive sugar info from registering an SoaS
113def get_identifier_info(info):
114  identifier_dir = os.path.expanduser('~') + '/.sugar/default/identifiers/'
115  fh = open(os.path.join(identifier_dir, info),'r')
116  data = fh.read().rstrip('\0\n')
117  fh.close()
118  return data
119
120# if run directly as script
121if __name__ == "__main__":
122
123    if have_ofw_tree():
124       sn = read_ofw('mfg-data/SN')
125       BACKUP_SERVER = 'schoolserver'
126    else:
127      # retrieve the serial and backup_url
128      sn = get_identifier_info('sn')
129      BACKUP_SERVER = get_identifier_info('backup_url')
130     
131    backup_url = 'http://'+BACKUP_SERVER+'/backup/1'
132
133    ds_path = env.get_profile_path('datastore')
134    pk_path = os.path.join(env.get_profile_path(), 'owner.key')
135
136    # Check backup server availability.
137    # On 503 ("too busy") apply exponential back-off
138    # over 10 attempts. Combined with the staggered sleep
139    # in ds-backup.sh, this should keep thundering herds
140    # under control. We are also holding a flock to prevent
141    # local races.
142    # With range(1,7) we sleep up to 64 minutes.
143    for n in range(1,7):
144        sstatus = check_server_available(backup_url, sn)
145        if (sstatus == 200):
146            # cleared to run
147            rsync_to_xs(ds_path, BACKUP_SERVER+':datastore-current', pk_path, sn)
148            # this marks success to the controlling script...
149            os.system('touch ~/.sugar/default/ds-backup-done')
150            exit(0)
151        elif (sstatus == 503):
152            # exponenxtial backoff
153            time.sleep(60 * 2**n)
154        elif (sstatus == -1):
155            # could not connect - XS is not there
156            exit(1)
157        else:
158            # 500, 404, 403, or other unexpected value
159            exit(sstatus)
160