[Sugar-devel] [PATCH sugar-datastore 2/2] add (minimal) test suite (SL#1438)
Sascha Silbe
sascha-pgp at silbe.org
Wed Jan 19 15:38:31 EST 2011
Add a minimal test suite for the data store operating on public DBus API
level. Checks all public API calls, including some simple performance
measurements.
Signed-off-by: Sascha Silbe <sascha-pgp at silbe.org>
create mode 100644 tests/.gitignore
create mode 100644 tests/Makefile
create mode 100644 tests/__init__.py
create mode 100644 tests/basic_api_v2.txt
create mode 100755 tests/runalltests.py
create mode 100644 tests/test_massops.py
create mode 100644 tests/test_migration_v1_v2.py
diff --git a/Makefile.am b/Makefile.am
index bfebefe..d450f24 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,7 +1,3 @@
ACLOCAL_AMFLAGS = -I m4
-SUBDIRS = bin etc src
-
-test:
- @cd tests
- $(MAKE) -C tests test
+SUBDIRS = bin etc src tests
diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 0000000..2460008
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1 @@
+!Makefile
diff --git a/tests/Makefile b/tests/Makefile
new file mode 100644
index 0000000..cf9ac60
--- /dev/null
+++ b/tests/Makefile
@@ -0,0 +1,20 @@
+all:
+install:
+uninstall:
+
+check:
+ @./runalltests.py
+
+valgrind:
+ @echo "Profiling the process. Run kcachegrind on the output"
+ valgrind --tool=callgrind --suppressions=valgrind-python.supp python runalltests.py
+
+distclean: clean
+clean:
+ @find . -name "*.pyc" -exec rm {} \;
+ @find . -name "*.pyo" -exec rm {} \;
+ @find . -name "*~" -exec rm {} \;
+ @find . -name "callgrind.out*" -exec rm {} \;
+
+tags:
+
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..5b3912c
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1 @@
+# testing package
diff --git a/tests/basic_api_v2.txt b/tests/basic_api_v2.txt
new file mode 100644
index 0000000..15d4cd6
--- /dev/null
+++ b/tests/basic_api_v2.txt
@@ -0,0 +1,135 @@
+>>> import os
+>>> import tempfile
+>>> import time
+
+Define some helper functions
+>>> def test_unique(items):
+... return not [True for e in items if items.count(e) > 1]
+>>> def to_native(value):
+... if isinstance(value, list):
+... return [to_native(e) for e in value]
+... elif isinstance(value, dict):
+... return dict([(to_native(k), to_native(v)) for k, v in value.items()])
+... elif isinstance(value, unicode):
+... return unicode(value)
+... elif isinstance(value, str):
+... return str(value)
+... return value
+
+
+Connect to datastore using DBus and wait for it to get ready:
+>>> import dbus
+>>> DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore'
+>>> DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore'
+>>> DS_DBUS_PATH = '/org/laptop/sugar/DataStore'
+>>> bus = dbus.SessionBus()
+>>> ds = dbus.Interface(bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH), DS_DBUS_INTERFACE)
+
+
+Make sure we're starting from an empty datastore:
+>>> assert ds.find({}, [], byte_arrays=True) == ([], 0)
+
+
+Create something to play with:
+>>> o1_uid = ds.create({'title': 'DS test object 1', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1'}, '', False)
+>>> assert isinstance(o1_uid, basestring)
+>>> o2_uid = ds.create({'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '', False)
+>>> assert isinstance(o2_uid, basestring)
+>>> o3_uid = ds.create({'title': 'DS test object 3', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '', False)
+>>> assert isinstance(o3_uid, basestring)
+>>> assert test_unique([o1_uid, o2_uid, o3_uid])
+
+
+Check everything is there:
+>>> assert sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])) == \
+... [{u'title': 'DS test object 1', u'activity': 'org.sugarlabs.DataStoreTest1'},
+... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest2'},
+... {u'title': 'DS test object 3', u'activity': 'org.sugarlabs.DataStoreTest2'}]
+>>> ds.get_filename(o1_uid, byte_arrays=True)
+dbus.String(u'')
+>>> ds.get_filename(o2_uid, byte_arrays=True)
+dbus.String(u'')
+>>> ds.get_filename(o3_uid, byte_arrays=True)
+dbus.String(u'')
+
+
+
+Test get_uniquevaluesfor().
+>>> sorted(ds.get_uniquevaluesfor('activity', {}))
+[dbus.String(u'org.sugarlabs.DataStoreTest1'), dbus.String(u'org.sugarlabs.DataStoreTest2')]
+
+
+Change some entries:
+>>> ds.update(o1_uid, {'title': 'DS test object 1 updated', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'foo'}, '', False)
+>>> ds.update(o2_uid, {'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'bar baz'}, '', False)
+>>> ds.update(o3_uid, {'title': 'DS test object 2', 'mime_type': 'text/html', 'activity': 'org.sugarlabs.DataStoreTest3', 'timestamp': 10000}, '', False)
+>>> assert sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])) == \
+... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'},
+... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'},
+... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}]
+
+Retrieve metadata for a single entry, ignoring variable data:
+>>> d=dict(ds.get_properties(o3_uid, byte_arrays=True))
+>>> del d['uid'], d['timestamp'], d['creation_time']
+>>> assert to_native(d) == {u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}
+
+
+Find entries using "known" metadata:
+>>> assert sorted(to_native(ds.find({'mime_type': ['text/plain']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])) == \
+... [{u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'bar baz'},
+... {u'title': 'DS test object 1 updated', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'foo'}]
+>>> assert sorted(to_native(ds.find({'mime_type': ['text/html']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])) == \
+... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}]
+>>> assert sorted(to_native(ds.find({'uid': o3_uid}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0])) == \
+... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}]
+>>> assert sorted(to_native(ds.find({'timestamp': (9000, 11000)}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0])) == \
+... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}]
+
+Find entries using "unknown" metadata (=> returns all entries):
+>>> assert sorted(to_native(ds.find({'title': 'DS test object 2'}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])) == \
+... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'},
+... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'bar baz'},
+... {u'title': 'DS test object 1 updated', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'foo'}]
+
+You can specify a (primary) sort order. Please note that the secondary sort order is undefined / implementation-dependent.
+>>> assert to_native(ds.find({'order_by': ['+title']}, ['title', 'activity'], byte_arrays=True)[0]) == \
+... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'},
+... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'},
+... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}]
+>>> assert to_native(ds.find({'order_by': ['-title']}, ['title', 'activity'], byte_arrays=True)[0]) == \
+... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'},
+... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'},
+... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}]
+
+Delete an entry:
+>>> ds.delete(o1_uid)
+>>> assert sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])) == \
+... [{u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1'},
+... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest3'}]
+
+
+Create an entry with content:
+>>> dog_content = 'The quick brown dog jumped over the lazy fox.'
+>>> dog_props = {'title': 'dog/fox story', 'mime_type': 'text/plain'}
+>>> dog_file = tempfile.NamedTemporaryFile()
+>>> dog_file.write(dog_content)
+>>> dog_file.flush()
+>>> dog_uid = ds.create(dog_props, dog_file.name, False)
+
+Retrieve and verify the entry with content:
+>>> dog_retrieved = ds.get_filename(dog_uid)
+>>> assert(file(dog_retrieved).read() == dog_content)
+>>> os.remove(dog_retrieved)
+
+Update the entry content:
+>>> dog_content = 'The quick brown fox jumped over the lazy dog.'
+>>> dog_file.seek(0)
+>>> dog_file.write(dog_content)
+>>> dog_file.flush()
+>>> ds.update(dog_uid, dog_props, dog_file.name, False)
+
+Verify updated content:
+>>> dog_retrieved = ds.get_filename(dog_uid)
+>>> assert(file(dog_retrieved).read() == dog_content)
+>>> os.remove(dog_retrieved)
+>>> dog_file.close()
diff --git a/tests/runalltests.py b/tests/runalltests.py
new file mode 100755
index 0000000..9a7c8f8
--- /dev/null
+++ b/tests/runalltests.py
@@ -0,0 +1,360 @@
+#!/usr/bin/env python
+"""Run all tests in the current directory.
+
+You can either call it without arguments to run all tests or name specific
+ones to run:
+
+ ./runalltests.py test_massops.py
+"""
+
+import doctest
+import errno
+import logging
+from optparse import OptionParser
+import os
+import os.path
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+import dbus
+import dbus.mainloop.glib
+import gobject
+
+
+logging.basicConfig(level=logging.WARN,
+ format='%(asctime)-15s %(name)s %(levelname)s:'
+ ' %(message)s',
+ stream=sys.stderr)
+
+
+DOCTESTS = [
+ 'basic_api_v2.txt',
+]
+DOCTEST_OPTIONS = doctest.ELLIPSIS
+DOCTEST_OPTIONS |= doctest.REPORT_ONLY_FIRST_FAILURE
+
+DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore'
+DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore'
+DS_DBUS_PATH = '/org/laptop/sugar/DataStore'
+
+ENVIRONMENT_WHITELIST = [
+ 'LD_LIBRARY_PATH',
+ 'MALLOC_CHECK_',
+ 'MASSOPS_RUNS',
+ 'SUGAR_LOGGER_LEVEL',
+]
+
+SERVICE_TEMPLATE = """
+[D-BUS Service]
+Name = org.laptop.sugar.DataStore
+Exec = %s/bin/datastore-service
+"""
+
+
+def setup():
+ """Prepare for testing and return environment.
+
+ Sets HOME and creates a new process group so we can clean up easily later.
+ Sets up environment variables and imports whitelisted ones.
+ """
+ environment = {}
+ for name in ENVIRONMENT_WHITELIST:
+ if name in os.environ:
+ environment[name] = os.environ[name]
+
+ environment['HOME'] = tempfile.mkdtemp(prefix='datastore-test')
+ if 'PYTHONPATH' in os.environ:
+ python_path = os.environ.get('PYTHONPATH').split(':')
+ else:
+ python_path = []
+
+ # Run tests on sources instead of on installed files.
+ basedir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), '..')
+ python_path = [os.path.join(basedir, 'src')] + python_path
+ environment['PYTHONPATH'] = ':'.join(python_path)
+ environment['PATH'] = ':'.join([os.path.join(basedir, 'bin'),
+ os.environ['PATH']])
+
+ service_dir = os.path.join(environment['HOME'], 'dbus-1', 'services')
+ service_path = os.path.join(service_dir,
+ 'org.laptop.sugar.DataStore.service')
+ os.makedirs(service_dir)
+ service_file = file(service_path, 'w')
+ service_file.write(SERVICE_TEMPLATE % (basedir, ))
+ service_file.close()
+ environment['XDG_DATA_DIRS'] = environment['HOME']
+
+ os.setpgid(0, 0)
+ # prevent suicide in cleanup()
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ return environment
+
+
+def wait_children():
+ """Wait for all children to exit."""
+ try:
+ while True:
+ os.wait()
+
+ except OSError, exception:
+ if exception.errno in (errno.ECHILD, errno.ESRCH):
+ # ECHILD is not documented in kill(2) and kill(2p), but used by
+ # Linux to indicate no child processes remaining to be waited for
+ return
+
+ raise
+
+
+def cleanup(home, keep_files, dbus_pid):
+ """Clean up test environment.
+
+ Kills all children and removes home directory.
+ """
+ if dbus_pid:
+ os.kill(-dbus_pid, signal.SIGTERM)
+
+ os.kill(0, signal.SIGTERM)
+ wait_children()
+
+ if not keep_files:
+ shutil.rmtree(home)
+
+
+class TestSuiteWrapper(unittest.TestCase):
+ """Wrap a test suite to clean up after it.
+
+ This ensures each test module gets a clean data store instance.
+ """
+
+ def __init__(self, suite):
+ self._wrapped_suite = suite
+ self._bus = dbus.SessionBus()
+ self._loop = None
+ unittest.TestCase.__init__(self)
+
+ def runTest(self, result=None):
+ self._wrapped_suite(result)
+
+ def run(self, result=None):
+ if result is None:
+ result = self.defaultTestResult()
+ result.startTest(self)
+ try:
+ try:
+ self.setUp()
+ except KeyboardInterrupt:
+ raise
+ except:
+ result.addError(self, self._exc_info())
+ return
+
+ ok = False
+ try:
+ self.runTest(result)
+ ok = True
+ except self.failureException:
+ result.addFailure(self, self._exc_info())
+ except KeyboardInterrupt:
+ raise
+ except:
+ result.addError(self, self._exc_info())
+
+ try:
+ self.tearDown()
+ except KeyboardInterrupt:
+ raise
+ except:
+ result.addError(self, self._exc_info())
+ ok = False
+ if ok:
+ result.addSuccess(self)
+ finally:
+ result.stopTest(self)
+
+ def shortDescription(self):
+ doc = self._wrapped_suite.__doc__
+ return doc and doc.split('\n')[0].strip() or None
+
+ def tearDown(self):
+ self._kill_data_store()
+ self._clean_data_store()
+
+ def _kill_data_store(self):
+ pgrep = subprocess.Popen(['pgrep', '-g', os.environ['DBUS_PID'],
+ '-f', 'datastore-service'],
+ close_fds=True, stdout=subprocess.PIPE)
+ stdout, stderr_ = pgrep.communicate()
+ pids = stdout.strip().split('\n')
+ if len(pids) != 1 or not pids[0]:
+ raise ValueError("Can't find (a single) data store process "
+ "(pgrep output %r)" % (stdout, ))
+
+ pid = int(pids[0])
+ self._loop = gobject.MainLoop()
+ self._bus.watch_name_owner(DS_DBUS_SERVICE, self._service_changed_cb)
+ os.kill(pid, signal.SIGTERM)
+ self._loop.run()
+
+ def _service_changed_cb(self, new_owner):
+ if not new_owner:
+ self._loop.quit()
+
+ def _clean_data_store(self):
+ profile = os.environ.get('SUGAR_PROFILE', 'default')
+ base_dir = os.path.join(os.path.expanduser('~'), '.sugar', profile)
+ root_path = os.path.join(base_dir, 'datastore')
+ shutil.rmtree(root_path)
+
+
+class TimedTestResult(unittest._TextTestResult):
+ """Store and display test results and test runtime.
+
+ Only displays actual tests, not test suite wrappers."""
+
+ # Depending on a private class is bad style, but the only alternative is
+ # copying it verbatim.
+ # pylint: disable=W0212
+
+ def __init__(self, stream, descriptions, verbosity):
+ unittest._TextTestResult.__init__(self, stream, descriptions,
+ verbosity)
+ self.start_times = {}
+ self.run_times = {}
+
+ def startTest(self, test):
+ self.start_times[test] = time.time()
+ unittest.TestResult.startTest(self, test)
+ if not self.showAll:
+ return
+
+ description = self.getDescription(test)
+ if isinstance(test, TestSuiteWrapper):
+ self.stream.write('Test Suite: %s\n' % (description, ))
+ else:
+ self.stream.write(' %s ... ' % (description, ))
+
+ def stopTest(self, test):
+ if test in self.start_times and test not in self.run_times:
+ self.run_times[test] = time.time() - self.start_times[test]
+
+ unittest._TextTestResult.stopTest(self, test)
+
+ def addSuccess(self, test):
+ if test in self.start_times and test not in self.run_times:
+ self.run_times[test] = time.time() - self.start_times[test]
+
+ run_time = self.run_times.get(test, -1)
+
+ unittest.TestResult.addSuccess(self, test)
+
+ if isinstance(test, TestSuiteWrapper):
+ return
+
+ if self.showAll:
+ self.stream.writeln('ok (%.3fs)' % (run_time, ))
+ elif self.dots:
+ self.stream.write('.')
+
+
+class TimedTestRunner(unittest.TextTestRunner):
+ """Run tests, displaying test result and runtime in textual form."""
+
+ def _makeResult(self):
+ return TimedTestResult(self.stream, self.descriptions, self.verbosity)
+
+
+def test_suite(tests=None):
+ suite = unittest.TestSuite()
+ if not tests:
+ test_dir = os.path.dirname(__file__)
+ tests = DOCTESTS
+ tests += [name for name in os.listdir(test_dir)
+ if name.startswith('test') and name.endswith('.py')]
+
+ for test in tests:
+ if test.endswith('.txt'):
+ doc_suite = doctest.DocFileSuite(test, optionflags=DOCTEST_OPTIONS)
+ doc_suite.__doc__ = test
+ suite.addTest(TestSuiteWrapper(doc_suite))
+
+ elif test.endswith('.py'):
+ m = __import__(test[:-3])
+ if hasattr(m, 'suite'):
+ suite.addTest(TestSuiteWrapper(m.suite()))
+
+ return suite
+
+
+def run_tests(tests):
+ dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
+ runner = TimedTestRunner(verbosity=2)
+ suite = test_suite(tests)
+ result = runner.run(suite)
+ if result.wasSuccessful():
+ return 0
+ else:
+ return 10
+
+
+def _start_dbus(environment):
+ pipe = subprocess.Popen(['dbus-launch'], stdout=subprocess.PIPE,
+ close_fds=True, env=environment,
+ cwd=environment['HOME'])
+ stdout, stderr_ = pipe.communicate()
+ pid = None
+ address = None
+ for line in stdout.strip().split('\n'):
+ key, value = line.split('=', 1)
+ if key == 'DBUS_SESSION_BUS_ADDRESS':
+ address = value
+ elif key == 'DBUS_SESSION_BUS_PID':
+ pid = int(value)
+ else:
+ raise ValueError('Cannot parse dbus-launch output: %r' % (line, ))
+
+ assert pid is not None
+ assert address is not None
+ return pid, address
+
+
+def _parse_options():
+ """Parse command line arguments."""
+ parser = OptionParser()
+ parser.add_option('-k', '--keep', dest='keep',
+ action='store_true', default=False,
+ help='Keep temporary files')
+ parser.add_option('', '--stage2', dest='stage2',
+ action='store_true', default=False,
+ help='For internal use only')
+ return parser.parse_args()
+
+
+def main(my_name, arguments):
+ options, tests = _parse_options()
+ if not options.stage2:
+ environment = setup()
+ dbus_pid = None
+ dbus_address = None
+ try:
+ dbus_pid, dbus_address = _start_dbus(environment)
+ environment['DBUS_SESSION_BUS_ADDRESS'] = dbus_address
+ environment['DBUS_PID'] = str(dbus_pid)
+
+ pipe = subprocess.Popen([os.path.abspath(my_name),
+ '--stage2'] + arguments,
+ cwd=environment['HOME'], env=environment)
+ return pipe.wait()
+
+ finally:
+ cleanup(environment['HOME'], options.keep, dbus_pid)
+
+ return run_tests(tests)
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[0], sys.argv[1:]))
diff --git a/tests/test_massops.py b/tests/test_massops.py
new file mode 100644
index 0000000..edce0c3
--- /dev/null
+++ b/tests/test_massops.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+"""Large number of operations intended for measuring performance."""
+
+import dbus
+import decorator
+import os
+import tempfile
+import time
+import unittest
+
+
+DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore'
+DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore'
+DS_DBUS_PATH = '/org/laptop/sugar/DataStore'
+NUM_RUNS = int(os.environ.get('MASSOPS_RUNS', '100'))
+IGNORE_PROPERTIES = [
+ 'checksum',
+ 'creation_time',
+ 'number',
+ 'timestamp',
+ 'uid',
+]
+
+
+ at decorator.decorator
+def repeat(func, *args, **kwargs):
+ """Run the decorated function NUM_RUNS times."""
+ for i_ in range(NUM_RUNS):
+ func(*args, **kwargs)
+
+
+class MassOpsTestCase(unittest.TestCase):
+ """Large number of operations intended for measuring performance."""
+
+ def setUp(self):
+ # pylint: disable=C0103
+ self._bus = dbus.SessionBus()
+ self._datastore = dbus.Interface(self._bus.get_object(DS_DBUS_SERVICE,
+ DS_DBUS_PATH), DS_DBUS_INTERFACE)
+
+ _create_properties = {
+ 'title': 'DS test object',
+ 'mime_type': 'text/plain',
+ 'activity': 'org.sugarlabs.DataStoreTest1',
+ }
+ _create_content = 'Foo bar\n' * 1000
+
+ def test_create(self):
+ """Run create() lots of times to create new objects."""
+ for i in range(NUM_RUNS):
+ content_file = tempfile.NamedTemporaryFile()
+ content_file.write(self._create_content)
+ content_file.flush()
+ properties = self._create_properties.copy()
+ properties['number'] = str(i)
+ properties['timestamp'] = time.time()
+ self._datastore.create(properties, content_file.name, False)
+ content_file.close()
+
+ @repeat
+ def test_find_all(self):
+ """Run find() to list all entries."""
+ entries, total_count = self._datastore.find({}, ['number'],
+ byte_arrays=True)
+ self.assertEquals(total_count, NUM_RUNS)
+ self.assertEquals(total_count, len(entries))
+ for position, entry in enumerate(entries):
+ self.assertEquals(int(entry['number']), NUM_RUNS - position - 1)
+
+ @repeat
+ def test_find_all_reverse_time(self):
+ """Run find() to list all entries in reverse chronological order."""
+ entries, total_count = \
+ self._datastore.find({'order_by': ['-timestamp']}, ['number'],
+ byte_arrays=True)
+ self.assertEquals(total_count, NUM_RUNS)
+ self.assertEquals(total_count, len(entries))
+ for position, entry in enumerate(entries):
+ self.assertEquals(int(entry['number']), position)
+
+ @repeat
+ def test_find_all_title(self):
+ """Run find() to list all entries ordered by title."""
+ entries, total_count = \
+ self._datastore.find({'order_by': ['+title']}, ['tree_id'],
+ byte_arrays=True)
+ self.assertEquals(total_count, NUM_RUNS)
+ self.assertEquals(total_count, len(entries))
+
+ @repeat
+ def test_find_all_reverse_title(self):
+ """Run find() to list all entries ordered by title (reversed)."""
+ entries, total_count = \
+ self._datastore.find({'order_by': ['-title']}, ['tree_id'],
+ byte_arrays=True)
+ self.assertEquals(total_count, NUM_RUNS)
+ self.assertEquals(total_count, len(entries))
+
+ @repeat
+ def test_find_all_chunked(self):
+ """Run find() to list all entries in small chunks."""
+ chunk_size = 30
+ for chunk_start in range(0, NUM_RUNS, 30):
+ entries, total_count = \
+ self._datastore.find({'offset': chunk_start,
+ 'limit': chunk_size}, ['number'],
+ byte_arrays=True)
+ self.assertEquals(len(entries),
+ min(chunk_size, NUM_RUNS - chunk_start))
+ self.assertEquals(total_count, NUM_RUNS)
+ for position, entry in enumerate(entries):
+ self.assertEquals(int(entry['number']),
+ NUM_RUNS - (chunk_start + position) - 1)
+
+ def test_get_properties(self):
+ """Run get_properties() on all entries and verify result."""
+ for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]:
+ properties = \
+ self._datastore.get_properties(entry['uid'], byte_arrays=True)
+ self.assertEquals(properties.pop('filesize'),
+ str(len(self._create_content)))
+ self._filter_properties(properties)
+ self.assertEquals(properties, self._create_properties)
+
+ def test_get_filename(self):
+ """Run get_filename() on all entries and verify content."""
+ for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]:
+ filename = self._datastore.get_filename(entry['uid'],
+ byte_arrays=True)
+ try:
+ self.assertEquals(file(filename).read(), self._create_content)
+ finally:
+ os.remove(filename)
+
+ _update_properties = {
+ 'title': 'DS test object (updated)',
+ 'mime_type': 'text/plain',
+ 'activity': 'org.sugarlabs.DataStoreTest1',
+ }
+ _update_content = 'Foo bar baz\n' * 1000
+
+ def test_update(self):
+ """Update the content of all existing entries"""
+ content_file = tempfile.NamedTemporaryFile()
+ content_file.write(self._update_content)
+ content_file.flush()
+ for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]:
+ self._datastore.update(entry['uid'], self._update_properties,
+ content_file.name, False)
+
+ def test_update_verify(self):
+ """
+ Verify test_update() has changed content and metadata of all entries.
+ """
+ for entry in self._datastore.find({}, [], byte_arrays=True)[0]:
+ filename = self._datastore.get_filename(entry['uid'],
+ byte_arrays=True)
+ self.assertEquals(entry.pop('filesize'),
+ str(len(self._update_content)))
+ self._filter_properties(entry)
+ try:
+ self.assertEquals(entry, self._update_properties)
+ self.assertEquals(file(filename).read(), self._update_content)
+ finally:
+ os.remove(filename)
+
+ def _filter_properties(self, properties):
+ for key in IGNORE_PROPERTIES:
+ properties.pop(key, None)
+
+
+def suite():
+ test_suite = unittest.TestLoader().loadTestsFromTestCase(MassOpsTestCase)
+ test_suite.__doc__ = MassOpsTestCase.__doc__
+ return test_suite
diff --git a/tests/test_migration_v1_v2.py b/tests/test_migration_v1_v2.py
new file mode 100644
index 0000000..bad1009
--- /dev/null
+++ b/tests/test_migration_v1_v2.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+"""Test datastore migration from version 1 to version 2."""
+
+import dbus
+import hashlib
+import os
+import time
+import unittest
+import uuid
+
+
+DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore'
+DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore'
+DS_DBUS_PATH = '/org/laptop/sugar/DataStore'
+IGNORE_PROPERTIES = [
+ 'activity_id',
+ 'checksum',
+ 'creation_time',
+ 'ctime',
+ 'mtime',
+ 'number',
+ 'timestamp',
+ 'uid',
+]
+
+
+class MigrationV1V2TestCase(unittest.TestCase):
+ """Test datastore migration from version 1 to version 2."""
+
+ def __init__(self, *args, **kwargs):
+ unittest.TestCase.__init__(self, *args, **kwargs)
+ self._templates = self._v1_properties * 10
+
+ def setUp(self):
+ # pylint: disable=C0103
+ profile = os.environ.get('SUGAR_PROFILE', 'default')
+ base_dir = os.path.join(os.path.expanduser('~'), '.sugar', profile)
+ self._root_path = os.path.join(base_dir, 'datastore')
+ if not os.path.exists(self._root_path):
+ self._create_v1_datastore()
+
+ self._bus = dbus.SessionBus()
+ self._datastore = dbus.Interface(self._bus.get_object(DS_DBUS_SERVICE,
+ DS_DBUS_PATH),
+ DS_DBUS_INTERFACE)
+
+ _v1_properties = [
+ {
+ 'title': lambda number: 'DS test object %d' % (number, ),
+ 'mime_type': 'text/plain',
+ },
+ {
+ 'title': lambda number: 'DS test object %d' % (number, ),
+ 'mime_type': 'text/html',
+ },
+ {
+ 'title': lambda number: 'DS test object %d' % (number, ),
+ 'title_set_by_user': '1',
+ 'keep': '1',
+ 'mime_type': 'text/html',
+ 'activity': 'org.sugarlabs.DataStoreTest2',
+ 'activity_id': lambda number_: str(uuid.uuid4()),
+ 'timestamp': lambda number_: time.time(),
+ 'icon-color': '#00ff00,#0000ff',
+ 'buddies': '{}',
+ 'description': 'DS migration test object',
+ 'tags': lambda number: 'test tag%d' % (number, ),
+ 'preview': dbus.ByteArray(''.join([chr(i) for i in range(255)])),
+ },
+ {
+ 'title': lambda number: 'DS test object %d' % (number, ),
+ 'activity': 'org.sugarlabs.DataStoreTest3',
+ 'activity_id': lambda number_: str(uuid.uuid4()),
+ 'ctime': lambda number_: time.strftime('%Y-%m-%dT%H:%M:%S'),
+ },
+ {
+ 'title': lambda number: 'DS test object %d' % (number, ),
+ 'activity': 'org.sugarlabs.DataStoreTest4',
+ 'activity_id': lambda number_: str(uuid.uuid4()),
+ 'mtime': lambda number_: time.strftime('%Y-%m-%dT%H:%M:%S'),
+ },
+ {},
+ ]
+
+ def _v1_content(self, num):
+ return ('Foo bar %d\n' % (num, )) * 1000
+
+ def _create_v1_datastore(self):
+ """Create a version 1 datastore on disk."""
+ os.makedirs(self._root_path)
+ file(os.path.join(self._root_path, 'version'), 'w').write('1')
+ for i, template in enumerate(self._templates):
+ metadata = self._fill_template(template, i)
+ data = self._v1_content(i)
+ tree_id = str(uuid.uuid4())
+ metadata['uid'] = tree_id
+ metadata['number'] = i
+
+ self._create_v1_entry(tree_id, metadata, data)
+
+ def _fill_template(self, template, i):
+ metadata = {}
+ for (key, value) in template.items():
+ if callable(value):
+ value = value(i)
+
+ metadata[key] = value
+
+ return metadata
+
+ def _create_v1_entry(self, tree_id, metadata, data):
+ """Create a single version 1 datastore entry."""
+ checksum = hashlib.md5(data).hexdigest()
+ entry_dir = os.path.join(self._root_path, tree_id[:2], tree_id)
+ os.makedirs(entry_dir)
+ file(os.path.join(entry_dir, 'data'), 'w').write(data)
+ self._write_v1_metadata(os.path.join(entry_dir, 'metadata'), metadata)
+ checksum_dir = os.path.join(self._root_path, 'checksums', checksum)
+ os.makedirs(checksum_dir)
+ file(os.path.join(checksum_dir, tree_id), 'w').close()
+
+ def _write_v1_metadata(self, directory, metadata):
+ os.makedirs(directory)
+ for key, value in metadata.items():
+ file(os.path.join(directory, key), 'w').write(str(value))
+
+ def test_find_all(self):
+ """Run find() to list all migrated entries."""
+ entries_, count = self._find({}, ['uid'])
+ self.assertEquals(count, len(self._templates))
+
+ def test_get_properties(self):
+ """Run get_properties() on all entries and verify result."""
+ for entry in self._find({}, ['uid'])[0]:
+ properties = self._datastore.get_properties(entry['uid'],
+ byte_arrays=True)
+ number = int(properties['number'])
+ expected = self._fill_template(self._templates[number],
+ number)
+ self.assertEquals(properties.pop('filesize'),
+ str(len(self._v1_content(number))))
+ self._filter_properties(properties)
+ self._filter_properties(expected)
+ self.assertEquals(properties, expected)
+
+ def test_get_filename(self):
+ """Run get_filename() on all entries and verify content."""
+ for entry in self._find({}, ['number', 'uid'])[0]:
+ filename = self._datastore.get_filename(entry['uid'],
+ byte_arrays=True)
+ content = file(filename).read()
+ os.remove(filename)
+ number = int(entry['number'])
+ expected = self._v1_content(number)
+ self.assertEquals(content, expected)
+
+ def _find(self, query, properties):
+ return self._datastore.find(dbus.Dictionary(query, signature='sv'),
+ properties, byte_arrays=True)
+
+ def _filter_properties(self, properties):
+ for key in IGNORE_PROPERTIES:
+ properties.pop(key, None)
+
+
+def suite():
+ test_loader = unittest.TestLoader()
+ test_suite = test_loader.loadTestsFromTestCase(MigrationV1V2TestCase)
+ test_suite.__doc__ = MigrationV1V2TestCase.__doc__
+ return test_suite
--
1.7.2.3
More information about the Sugar-devel
mailing list