diff --git a/superlance/httpok.py b/superlance/httpok.py index 1839bf4..b86b185 100644 --- a/superlance/httpok.py +++ b/superlance/httpok.py @@ -105,6 +105,7 @@ import socket import sys import time +import datetime from superlance.compat import urlparse from superlance.compat import xmlrpclib @@ -232,7 +233,7 @@ def act(self, subject, msg): messages = [msg] def write(msg): - self.stderr.write('%s\n' % msg) + self.stderr.write('%s %s\n' % (datetime.datetime.now().isoformat(), msg)) self.stderr.flush() messages.append(msg) @@ -284,7 +285,7 @@ def mail(self, email, subject, msg): body += msg with os.popen(self.sendmail, 'w') as m: m.write(body) - self.stderr.write('Mailed:\n\n%s' % body) + self.stderr.write('%s Mailed:\n\n%s' % (datetime.datetime.now().isoformat(), body)) self.mailed = body def restart(self, spec, write): diff --git a/superlance/memmon.py b/superlance/memmon.py index 795ea7b..9c5a85c 100644 --- a/superlance/memmon.py +++ b/superlance/memmon.py @@ -84,6 +84,7 @@ import os import sys import time +import datetime from collections import namedtuple from superlance.compat import maxint from superlance.compat import xmlrpclib @@ -147,7 +148,8 @@ def runforever(self, test=False): if self.any is not None: status.append('Checking any=%s' % self.any) - self.stderr.write('\n'.join(status) + '\n') + self.stderr.write(datetime.datetime.now().isoformat() + ' ' + + '\n'.join(status) + '\n') infos = self.rpc.supervisor.getAllProcessInfo() @@ -170,19 +172,28 @@ def runforever(self, test=False): for n in name, pname: if n in self.programs: - self.stderr.write('RSS of %s is %s\n' % (pname, rss)) - if rss > self.programs[name]: + self.stderr.write('%s RSS of %s is %s\n' % + (datetime.datetime.now().isoformat(), + pname, + rss)) + if rss > self.programs[name]: self.restart(pname, rss) continue if group in self.groups: - self.stderr.write('RSS of %s is %s\n' % (pname, rss)) + self.stderr.write('%s RSS of %s is %s\n' % + (datetime.datetime.now().isoformat(), + pname, + rss)) if rss > self.groups[group]: self.restart(pname, rss) continue if self.any is not None: - self.stderr.write('RSS of %s is %s\n' % (pname, rss)) + self.stderr.write('%s RSS of %s is %s\n' % + (datetime.datetime.now().isoformat(), + pname, + rss)) if rss > self.any: self.restart(pname, rss) continue @@ -195,12 +206,14 @@ def runforever(self, test=False): def restart(self, name, rss): info = self.rpc.supervisor.getProcessInfo(name) uptime = info['now'] - info['start'] #uptime in seconds - self.stderr.write('Restarting %s\n' % name) + self.stderr.write('%s Restarting %s\n' % + (datetime.datetime.now().isoformat(), name)) + try: self.rpc.supervisor.stopProcess(name) except xmlrpclib.Fault as e: - msg = ('Failed to stop process %s (RSS %s), exiting: %s' % - (name, rss, e)) + msg = ('%s Failed to stop process %s (RSS %s), exiting: %s' % + (datetime.datetime.now().isoformat(), name, rss, e)) self.stderr.write(str(msg)) if self.email: subject = self.format_subject( @@ -212,8 +225,9 @@ def restart(self, name, rss): try: self.rpc.supervisor.startProcess(name) except xmlrpclib.Fault as e: - msg = ('Failed to start process %s after stopping it, ' - 'exiting: %s' % (name, e)) + msg = ('%s Failed to start process %s after stopping it, ' + 'exiting: %s' % + (datetime.datetime.now().isoformat(), name, e)) self.stderr.write(str(msg)) if self.email: subject = self.format_subject( diff --git a/superlance/tests/httpok_test.py b/superlance/tests/httpok_test.py index 175d5fb..2776a00 100644 --- a/superlance/tests/httpok_test.py +++ b/superlance/tests/httpok_test.py @@ -206,17 +206,15 @@ def test_runforever_eager_error_on_request_some(self): prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') - self.assertEqual(lines[0], - ("Restarting selected processes ['foo', 'bar', " - "'baz_01', 'notexisting']") - ) - self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') - self.assertEqual(lines[2], 'foo restarted') - self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') - self.assertEqual(lines[4], - 'baz:baz_01 not in RUNNING state, NOT restarting') - self.assertEqual(lines[5], - "Programs not restarted because they did not exist: ['notexisting']") + self.assertTrue("Restarting selected processes ['foo', 'bar', " + "'baz_01', 'notexisting']" in lines[0]) + self.assertTrue('foo is in RUNNING state, restarting' in lines[1]) + self.assertTrue('foo restarted' in lines[2]) + self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[3]) + self.assertTrue('baz:baz_01 not in RUNNING state, NOT restarting' in + lines[4]) + self.assertTrue("Programs not restarted because " + "they did not exist: ['notexisting']", lines[5]) mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 12) self.assertEqual(mailed[0], 'To: chrism@plope.com') @@ -231,12 +229,12 @@ def test_runforever_eager_error_on_request_any(self): prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') - self.assertEqual(lines[0], 'Restarting all running processes') - self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') - self.assertEqual(lines[2], 'foo restarted') - self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') - self.assertEqual(lines[4], - 'baz:baz_01 not in RUNNING state, NOT restarting') + self.assertTrue('Restarting all running processes' in lines[0]) + self.assertTrue('foo is in RUNNING state, restarting' in lines[1]) + self.assertTrue('foo restarted' in lines[2]) + self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[3]) + self.assertTrue('baz:baz_01 not in RUNNING state, NOT restarting' in + lines[4]) mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 11) self.assertEqual(mailed[0], 'To: chrism@plope.com') @@ -252,11 +250,11 @@ def test_runforever_eager_error_on_process_stop(self): prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') - self.assertEqual(lines[0], "Restarting selected processes ['FAILED']") - self.assertEqual(lines[1], 'foo:FAILED is in RUNNING state, restarting') - self.assertEqual(lines[2], - "Failed to stop process foo:FAILED: ") - self.assertEqual(lines[3], 'foo:FAILED restarted') + self.assertTrue("Restarting selected processes ['FAILED']" in lines[0]) + self.assertTrue('foo:FAILED is in RUNNING state, restarting' in lines[1]) + self.assertTrue("Failed to stop process foo:FAILED: " + "" in lines[2]) + self.assertTrue('foo:FAILED restarted' in lines[3]) mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 10) self.assertEqual(mailed[0], 'To: chrism@plope.com') @@ -272,12 +270,11 @@ def test_runforever_eager_error_on_process_start(self): prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') - self.assertEqual(lines[0], - "Restarting selected processes ['SPAWN_ERROR']") - self.assertEqual(lines[1], - 'foo:SPAWN_ERROR is in RUNNING state, restarting') - self.assertEqual(lines[2], - "Failed to start process foo:SPAWN_ERROR: ") + self.assertTrue("Restarting selected processes ['SPAWN_ERROR']" in lines[0]) + self.assertTrue('foo:SPAWN_ERROR is in RUNNING state, restarting' in + lines[1]) + self.assertTrue("Failed to start process foo:SPAWN_ERROR: " + "" in lines[2]) mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 9) self.assertEqual(mailed[0], 'To: chrism@plope.com') @@ -293,20 +290,18 @@ def test_runforever_eager_gcore(self): prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') - self.assertEqual(lines[0], - ("Restarting selected processes ['foo', 'bar', " - "'baz_01', 'notexisting']") - ) - self.assertEqual(lines[1], 'gcore output for foo:') + self.assertTrue("Restarting selected processes ['foo', 'bar', " + "'baz_01', 'notexisting']" in lines[0]) + self.assertTrue('gcore output for foo:' in lines[1]) self.assertEqual(lines[2], '') self.assertEqual(lines[3], ' ') - self.assertEqual(lines[4], 'foo is in RUNNING state, restarting') - self.assertEqual(lines[5], 'foo restarted') - self.assertEqual(lines[6], 'bar not in RUNNING state, NOT restarting') - self.assertEqual(lines[7], - 'baz:baz_01 not in RUNNING state, NOT restarting') - self.assertEqual(lines[8], - "Programs not restarted because they did not exist: ['notexisting']") + self.assertTrue('foo is in RUNNING state, restarting' in lines[4]) + self.assertTrue('foo restarted' in lines[5]) + self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[6]) + self.assertTrue('baz:baz_01 not in RUNNING state, NOT restarting' in + lines[7]) + self.assertTrue("Programs not restarted because they did not exist: " + "['notexisting']" in lines[8]) mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 15) self.assertEqual(mailed[0], 'To: chrism@plope.com') @@ -333,12 +328,10 @@ def test_runforever_not_eager_running(self): prog.stdin.seek(0) prog.runforever(test=True) lines = [x for x in prog.stderr.getvalue().split('\n') if x] - self.assertEqual(lines[0], - ("Restarting selected processes ['foo', 'bar']") - ) - self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') - self.assertEqual(lines[2], 'foo restarted') - self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') + self.assertTrue("Restarting selected processes ['foo', 'bar']" in lines[0]) + self.assertTrue('foo is in RUNNING state, restarting' in lines[1]) + self.assertTrue('foo restarted' in lines[2]) + self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[3]) mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 10) self.assertEqual(mailed[0], 'To: chrism@plope.com') @@ -368,12 +361,11 @@ def test_runforever_connrefused_error(self): prog.stdin.seek(0) prog.runforever(test=True) lines = [x for x in prog.stderr.getvalue().split('\n') if x] - self.assertEqual(lines[0], - ("Restarting selected processes ['foo', 'bar']") - ) - self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') - self.assertEqual(lines[2], 'foo restarted') - self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') + self.assertTrue("Restarting selected processes ['foo', 'bar']" + in lines[0]) + self.assertTrue('foo is in RUNNING state, restarting' in lines[1]) + self.assertTrue('foo restarted' in lines[2]) + self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[3]) mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 10) self.assertEqual(mailed[0], 'To: chrism@plope.com') diff --git a/superlance/tests/memmon_test.py b/superlance/tests/memmon_test.py index 5bc6e05..b936294 100644 --- a/superlance/tests/memmon_test.py +++ b/superlance/tests/memmon_test.py @@ -54,13 +54,13 @@ def test_runforever_tick_programs(self): memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 8) - self.assertEqual(lines[0], 'Checking programs bar=0, baz_01=0, foo=0') - self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') - self.assertEqual(lines[2], 'Restarting foo:foo') - self.assertEqual(lines[3], 'RSS of bar:bar is 2265088') - self.assertEqual(lines[4], 'Restarting bar:bar') - self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088') - self.assertEqual(lines[6], 'Restarting baz:baz_01') + self.assertTrue('Checking programs bar=0, baz_01=0, foo=0' in lines[0]) + self.assertTrue('RSS of foo:foo is 2264064' in lines[1]) + self.assertTrue('Restarting foo:foo' in lines[2]) + self.assertTrue('RSS of bar:bar is 2265088' in lines[3]) + self.assertTrue('Restarting bar:bar' in lines[4]) + self.assertTrue('RSS of baz:baz_01 is 2265088' in lines[5]) + self.assertTrue('Restarting baz:baz_01' in lines[6]) self.assertEqual(lines[7], '') mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) @@ -80,9 +80,9 @@ def test_runforever_tick_groups(self): memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 4) - self.assertEqual(lines[0], 'Checking groups foo=0') - self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') - self.assertEqual(lines[2], 'Restarting foo:foo') + self.assertTrue('Checking groups foo=0' in lines[0]) + self.assertTrue('RSS of foo:foo is 2264064' in lines[1]) + self.assertTrue('Restarting foo:foo' in lines[2]) self.assertEqual(lines[3], '') mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) @@ -102,13 +102,13 @@ def test_runforever_tick_any(self): memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 8) - self.assertEqual(lines[0], 'Checking any=0') - self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') - self.assertEqual(lines[2], 'Restarting foo:foo') - self.assertEqual(lines[3], 'RSS of bar:bar is 2265088') - self.assertEqual(lines[4], 'Restarting bar:bar') - self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088') - self.assertEqual(lines[6], 'Restarting baz:baz_01') + self.assertTrue('Checking any=0' in lines[0]) + self.assertTrue('RSS of foo:foo is 2264064' in lines[1]) + self.assertTrue('Restarting foo:foo' in lines[2]) + self.assertTrue('RSS of bar:bar is 2265088' in lines[3]) + self.assertTrue('Restarting bar:bar' in lines[4]) + self.assertTrue('RSS of baz:baz_01 is 2265088' in lines[5]) + self.assertTrue('Restarting baz:baz_01' in lines[6]) self.assertEqual(lines[7], '') mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) @@ -123,12 +123,12 @@ def test_runforever_tick_programs_and_groups(self): memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 7) - self.assertEqual(lines[0], 'Checking programs baz_01=0') - self.assertEqual(lines[1], 'Checking groups foo=0') - self.assertEqual(lines[2], 'RSS of foo:foo is 2264064') - self.assertEqual(lines[3], 'Restarting foo:foo') - self.assertEqual(lines[4], 'RSS of baz:baz_01 is 2265088') - self.assertEqual(lines[5], 'Restarting baz:baz_01') + self.assertTrue('Checking programs baz_01=0' in lines[0]) + self.assertTrue('Checking groups foo=0' in lines[1]) + self.assertTrue('RSS of foo:foo is 2264064' in lines[2]) + self.assertTrue('Restarting foo:foo' in lines[3]) + self.assertTrue('RSS of baz:baz_01 is 2265088' in lines[4]) + self.assertTrue('Restarting baz:baz_01' in lines[5]) self.assertEqual(lines[6], '') mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) @@ -148,8 +148,8 @@ def test_runforever_tick_programs_norestart(self): memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 3) - self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint) - self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') + self.assertTrue('Checking programs foo=%s' % maxint in lines[0]) + self.assertTrue('RSS of foo:foo is 2264064' in lines[1]) self.assertEqual(lines[2], '') self.assertEqual(memmon.mailed, False) @@ -163,8 +163,8 @@ def test_stopprocess_fault_tick_programs_norestart(self): memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 3) - self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint) - self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') + self.assertTrue('Checking programs foo=%s' % maxint in lines[0]) + self.assertTrue('RSS of foo:foo is 2264064' in lines[1]) self.assertEqual(lines[2], '') self.assertEqual(memmon.mailed, False) @@ -192,17 +192,17 @@ def test_stopprocess_fails_to_stop(self): self.assertRaises(xmlrpclib.Fault, memmon.runforever, True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 4) - self.assertEqual(lines[0], 'Checking programs BAD_NAME=%s' % 0) - self.assertEqual(lines[1], 'RSS of BAD_NAME:BAD_NAME is 2264064') - self.assertEqual(lines[2], 'Restarting BAD_NAME:BAD_NAME') - self.assertTrue(lines[3].startswith('Failed')) + self.assertTrue('Checking programs BAD_NAME=%s' % 0 in lines[0]) + self.assertTrue('RSS of BAD_NAME:BAD_NAME is 2264064' in lines[1]) + self.assertTrue('Restarting BAD_NAME:BAD_NAME' in lines[2]) + self.assertTrue('Failed' in lines[3]) mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: memmon: failed to stop process BAD_NAME:BAD_NAME, exiting') self.assertEqual(mailed[2], '') - self.assertTrue(mailed[3].startswith('Failed')) + self.assertTrue('Failed' in mailed[3]) def test_subject_no_name(self): """set the name to None to check if subject formats to: @@ -392,4 +392,3 @@ def test_argparser(self): if __name__ == '__main__': unittest.main() -