# -*- coding: utf-8 -*- # Copyright: 2016, Ableton AG, Berlin. All rights reserved. from __future__ import absolute_import, with_statement import random import unittest from itertools import count as icount import mox import xmlrpclib from abl.util import Bunch from abl.vpath.base import URI from abl.webconnector import ( CRASHLOG_EXT, PUSH2_EXT, WEBCONNECTOR_EXT, RPCConnector, clean_directory, upload_event_logs, ) from abl.webconnector.fileprocessing import pivot_lockfile from .common import FakeTransport, FakeUsageReporter, cleanup_memory_fs, create_file class NiceMockConnector(object): def upload_event_log(self, _content): return 0 def upload_log(self, _run_id, _kind, _content): pass class EventLoggingTests(unittest.TestCase): APP_BASE_PATH = "memory:///foo" AUTOUPDATES_DIR_BASE_PATH = "memory:///autoupdate_path" def setUp(self): super(EventLoggingTests, self).setUp() URI(self.APP_BASE_PATH).mkdir() URI(self.AUTOUPDATES_DIR_BASE_PATH).mkdir() def tearDown(self): cleanup_memory_fs() super(EventLoggingTests, self).tearDown() def test_old_file_deletion(self): test_file = URI(__file__).dirname() / "test_event.log" with test_file.open("r") as inf: content = inf.read() base = URI("memory:///") / "event_logs" base.mkdir() suffixes = [ "webconnector", "indexer", "crashlog", "swapper", "push2", "live+info", "push+info", ] def create_filenames(i, counter=icount()): prefix = "event_%i." % i res = [base / prefix + "log"] for suffix in suffixes: counting_suffix = "" if random.random() < 0.5 else "-%i" % counter.next() res.append(base / prefix + suffix + counting_suffix) return res count = 200 for i in xrange(count): for f in create_filenames(i): with f.open("w") as outf: outf.write(content) f._manipulate(mtime=f.mtime() + i) clean_directory(base) self.assertEqual(100 * (len(suffixes) + 1), len(base.listdir())) def test_old_file_deletion_with_crashlog(self): test_file = URI(__file__).dirname() / "test_event.log" with test_file.open("r") as inf: content = inf.read() base = URI("memory:///") / "event_logs" base.mkdir() count = 200 for i in xrange(count): f = base / ("event_%i.log" % i) with f.open("w") as outf: outf.write(content) mtime = f.mtime() + i f._manipulate(mtime=mtime) if not (i % 4): f = base / ("event_%i%s" % (i, CRASHLOG_EXT)) with f.open("w") as outf: outf.write("I'm a crashlog") f._manipulate(mtime=mtime) clean_directory(base) self.assertEqual(len(base.listdir()), 125) def test_old_file_deletion_respects_locks(self): test_file = URI(__file__).dirname() / "test_event.log" with test_file.open("r") as inf: content = inf.read() base = URI("memory:///") / "event_logs" base.mkdir() count = 200 for i in xrange(count): f = base / ("event_%i.log" % i) with f.open("w") as outf: outf.write(content) f._manipulate(mtime=f.mtime() + i) lockfile = pivot_lockfile(f) with lockfile.open("w") as outf: outf.write(".") lockfile._manipulate(mtime=f.mtime() + i, lock=True) clean_directory(base) self.assertEqual(len(base.listdir()), count * 2) def test_old_file_deletion_with_processed_logs(self): base = URI("memory:///") / "event_logs" base.mkdir() create_file(base / "event_log_1.log+processed") clean_directory(base, keep_latest=0) self.assertFalse(base.listdir()) def test_event_log_upload(self): test_file = URI(__file__).dirname() / "test_event.log" with test_file.open("r") as inf: content = inf.read() run_id = 1 mocker = mox.Mox() mock_conn = mocker.CreateMockAnything() mock_conn.upload_event_log(mox.IsA(xmlrpclib.Binary)).AndReturn(run_id) mocker.ReplayAll() connector = RPCConnector( Bunch( app_base_path=self.APP_BASE_PATH, autoupdates_dir_base_path=self.AUTOUPDATES_DIR_BASE_PATH, ), RPCConnector.DEFAULT_ENDPOINT, transport=FakeTransport(mock_conn), ) created_run_id = connector.upload_event_log(content) self.assertEqual(created_run_id, run_id) def test_crashlog_upload(self): run_id = 1 mocker = mox.Mox() mock_conn = mocker.CreateMockAnything() mock_conn.upload_log( run_id, CRASHLOG_EXT[1:], mox.IsA(xmlrpclib.Binary) ).AndReturn(True) mocker.ReplayAll() connector = RPCConnector( Bunch( app_base_path=EventLoggingTests.APP_BASE_PATH, autoupdates_dir_base_path=EventLoggingTests.AUTOUPDATES_DIR_BASE_PATH, ), RPCConnector.DEFAULT_ENDPOINT, transport=FakeTransport(mock_conn), ) res = connector.upload_log(run_id, CRASHLOG_EXT[1:], "crashlog content") assert res def test_event_log_dir_processing(self): base = URI("memory:///") / "event_logs" base.mkdir() session_id = "fake_session_id" event_1_content = "event_log_1" event_2_content = "event_log_2" event_3_content = "event_log_3" exts = sorted([CRASHLOG_EXT, WEBCONNECTOR_EXT, PUSH2_EXT + "-1"]) f = base / ("event_1.log") with f.open("w") as outf: outf.write(event_1_content) f._manipulate(mtime=f.mtime() + 1) for ext in exts: cl = base / ("event_1%s" % ext) with cl.open("w") as outf: outf.write("event 1 %s" % ext) cl._manipulate(mtime=f.mtime() + 1) f = base / ("event_2.log") with f.open("w") as outf: outf.write(event_2_content) f._manipulate(mtime=f.mtime() + 2) f = base / ("event_3.log") with f.open("w") as outf: outf.write(event_3_content) try: lockfile = pivot_lockfile(f) with lockfile.open("w") as outf: outf.write(" ") lockfile._manipulate(mtime=f.mtime() + 3, lock=True) mocker = mox.Mox() mock_conn = mocker.CreateMockAnything() run_id = 1 # we end up with exactly two calls # because the last file is locked mock_conn.upload_event_log(mox.IsA(xmlrpclib.Binary)).AndReturn(run_id) mock_conn.upload_event_log(mox.IsA(xmlrpclib.Binary)).AndReturn(run_id) for ext in exts: # one call to upload a crashlog mock_conn.upload_log( run_id, ext[1:], mox.IsA(xmlrpclib.Binary) ).AndReturn(True) mocker.ReplayAll() connector = RPCConnector( Bunch( app_base_path=EventLoggingTests.APP_BASE_PATH, autoupdates_dir_base_path=EventLoggingTests.AUTOUPDATES_DIR_BASE_PATH, ), RPCConnector.DEFAULT_ENDPOINT, session_id=session_id, transport=FakeTransport(mock_conn), ) upload_event_logs(connector, base, FakeUsageReporter()) mocker.VerifyAll() self.assertEqual( base.listdir(), [ "event_1.crashlog", "event_1.log+processed", "event_1.push2-1", "event_1.webconnector", "event_2.log+processed", "event_3.log", "event_3.log.lock", ], ) finally: lockfile._manipulate(unlock=True) def test_event_log_dir_processing_doesnt_touch_other_files(self): base = URI("memory:///") / "event_logs" base.mkdir() session_id = "fake_session_id" f = base / ("event_1.log") with f.open("w") as outf: outf.write("event_log_1") f = base / ("something.else") with f.open("w") as outf: outf.write("foobar") mocker = mox.Mox() mock_conn = mocker.CreateMockAnything() # we end up with exactly two calls # because the last file is locked mock_conn.upload_event_log(mox.IsA(xmlrpclib.Binary)).AndReturn(True) mocker.ReplayAll() connector = RPCConnector( Bunch( app_base_path=EventLoggingTests.APP_BASE_PATH, autoupdates_dir_base_path=EventLoggingTests.AUTOUPDATES_DIR_BASE_PATH, ), RPCConnector.DEFAULT_ENDPOINT, session_id=session_id, transport=FakeTransport(mock_conn), ) upload_event_logs(connector, base, FakeUsageReporter()) mocker.VerifyAll() self.assertEqual(base.listdir(), ["event_1.log+processed", "something.else"]) def test_event_log_dir_processing_doesnt_touch_phone_home_files(self): base = URI("memory:///") / "event_logs" base.mkdir() create_file(base / "event_1.log") create_file(base / "event_1.live+info") create_file(base / "event_1.push+info") connector = NiceMockConnector() upload_event_logs(connector, base, FakeUsageReporter()) self.assertEqual( base.listdir(), ["event_1.live+info", "event_1.log+processed", "event_1.push+info"], ) def test_event_log_dir_processing_marks_as_processed(self): base = URI("memory:///") / "event_logs" base.mkdir() create_file(base / "event_1.log") connector = NiceMockConnector() upload_event_logs(connector, base, FakeUsageReporter()) self.assertTrue((base / "event_1.log+processed").exists()) if __name__ == "__main__": unittest.main()