Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Include datetime in httpok and memmon script logs #75

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Include datetime in httpok and memmon script logs #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
10a3abc
include current datetime in the log files for httpOk
erral Feb 12, 2016
d705102
include current datetime in log files for memmon
erral Feb 12, 2016
ef53cd1
user assertTrue instead of assertIn, py26 doesn't have assertIn
erral Feb 12, 2016
246ae94
remove unused line
erral Feb 1, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions superlance/httpok.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@
import socket
import sys
import time
import datetime
from superlance.compat import urlparse
from superlance.compat import xmlrpclib

Expand Down Expand Up @@ -232,7 +233,7 @@ def act(self, subject, msg):
messages = [msg]

def write(msg):
self.stderr.write('%s\n' % msg)
self.stderr.write('%s %s\n' % (datetime.datetime.now().isoformat(), msg))
self.stderr.flush()
messages.append(msg)

Expand Down Expand Up @@ -284,7 +285,7 @@ def mail(self, email, subject, msg):
body += msg
with os.popen(self.sendmail, 'w') as m:
m.write(body)
self.stderr.write('Mailed:\n\n%s' % body)
self.stderr.write('%s Mailed:\n\n%s' % (datetime.datetime.now().isoformat(), body))
self.mailed = body

def restart(self, spec, write):
Expand Down
34 changes: 24 additions & 10 deletions superlance/memmon.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@
import os
import sys
import time
import datetime
from collections import namedtuple
from superlance.compat import maxint
from superlance.compat import xmlrpclib
Expand Down Expand Up @@ -147,7 +148,8 @@ def runforever(self, test=False):
if self.any is not None:
status.append('Checking any=%s' % self.any)

self.stderr.write('\n'.join(status) + '\n')
self.stderr.write(datetime.datetime.now().isoformat() + ' ' +
'\n'.join(status) + '\n')

infos = self.rpc.supervisor.getAllProcessInfo()

Expand All @@ -170,19 +172,28 @@ def runforever(self, test=False):

for n in name, pname:
if n in self.programs:
self.stderr.write('RSS of %s is %s\n' % (pname, rss))
if rss > self.programs[name]:
self.stderr.write('%s RSS of %s is %s\n' %
(datetime.datetime.now().isoformat(),
pname,
rss))
if rss > self.programs[name]:
self.restart(pname, rss)
continue

if group in self.groups:
self.stderr.write('RSS of %s is %s\n' % (pname, rss))
self.stderr.write('%s RSS of %s is %s\n' %
(datetime.datetime.now().isoformat(),
pname,
rss))
if rss > self.groups[group]:
self.restart(pname, rss)
continue

if self.any is not None:
self.stderr.write('RSS of %s is %s\n' % (pname, rss))
self.stderr.write('%s RSS of %s is %s\n' %
(datetime.datetime.now().isoformat(),
pname,
rss))
if rss > self.any:
self.restart(pname, rss)
continue
Expand All @@ -195,12 +206,14 @@ def runforever(self, test=False):
def restart(self, name, rss):
info = self.rpc.supervisor.getProcessInfo(name)
uptime = info['now'] - info['start'] #uptime in seconds
self.stderr.write('Restarting %s\n' % name)
self.stderr.write('%s Restarting %s\n' %
(datetime.datetime.now().isoformat(), name))

try:
self.rpc.supervisor.stopProcess(name)
except xmlrpclib.Fault as e:
msg = ('Failed to stop process %s (RSS %s), exiting: %s' %
(name, rss, e))
msg = ('%s Failed to stop process %s (RSS %s), exiting: %s' %
(datetime.datetime.now().isoformat(), name, rss, e))
self.stderr.write(str(msg))
if self.email:
subject = self.format_subject(
Expand All @@ -212,8 +225,9 @@ def restart(self, name, rss):
try:
self.rpc.supervisor.startProcess(name)
except xmlrpclib.Fault as e:
msg = ('Failed to start process %s after stopping it, '
'exiting: %s' % (name, e))
msg = ('%s Failed to start process %s after stopping it, '
'exiting: %s' %
(datetime.datetime.now().isoformat(), name, e))
self.stderr.write(str(msg))
if self.email:
subject = self.format_subject(
Expand Down
96 changes: 44 additions & 52 deletions superlance/tests/httpok_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,17 +206,15 @@ def test_runforever_eager_error_on_request_some(self):
prog.stdin.seek(0)
prog.runforever(test=True)
lines = prog.stderr.getvalue().split('\n')
self.assertEqual(lines[0],
("Restarting selected processes ['foo', 'bar', "
"'baz_01', 'notexisting']")
)
self.assertEqual(lines[1], 'foo is in RUNNING state, restarting')
self.assertEqual(lines[2], 'foo restarted')
self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting')
self.assertEqual(lines[4],
'baz:baz_01 not in RUNNING state, NOT restarting')
self.assertEqual(lines[5],
"Programs not restarted because they did not exist: ['notexisting']")
self.assertTrue("Restarting selected processes ['foo', 'bar', "
"'baz_01', 'notexisting']" in lines[0])
self.assertTrue('foo is in RUNNING state, restarting' in lines[1])
self.assertTrue('foo restarted' in lines[2])
self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[3])
self.assertTrue('baz:baz_01 not in RUNNING state, NOT restarting' in
lines[4])
self.assertTrue("Programs not restarted because "
"they did not exist: ['notexisting']", lines[5])
mailed = prog.mailed.split('\n')
self.assertEqual(len(mailed), 12)
self.assertEqual(mailed[0], 'To: chrism@plope.com')
Expand All @@ -231,12 +229,12 @@ def test_runforever_eager_error_on_request_any(self):
prog.stdin.seek(0)
prog.runforever(test=True)
lines = prog.stderr.getvalue().split('\n')
self.assertEqual(lines[0], 'Restarting all running processes')
self.assertEqual(lines[1], 'foo is in RUNNING state, restarting')
self.assertEqual(lines[2], 'foo restarted')
self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting')
self.assertEqual(lines[4],
'baz:baz_01 not in RUNNING state, NOT restarting')
self.assertTrue('Restarting all running processes' in lines[0])
self.assertTrue('foo is in RUNNING state, restarting' in lines[1])
self.assertTrue('foo restarted' in lines[2])
self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[3])
self.assertTrue('baz:baz_01 not in RUNNING state, NOT restarting' in
lines[4])
mailed = prog.mailed.split('\n')
self.assertEqual(len(mailed), 11)
self.assertEqual(mailed[0], 'To: chrism@plope.com')
Expand All @@ -252,11 +250,11 @@ def test_runforever_eager_error_on_process_stop(self):
prog.stdin.seek(0)
prog.runforever(test=True)
lines = prog.stderr.getvalue().split('\n')
self.assertEqual(lines[0], "Restarting selected processes ['FAILED']")
self.assertEqual(lines[1], 'foo:FAILED is in RUNNING state, restarting')
self.assertEqual(lines[2],
"Failed to stop process foo:FAILED: <Fault 30: 'FAILED'>")
self.assertEqual(lines[3], 'foo:FAILED restarted')
self.assertTrue("Restarting selected processes ['FAILED']" in lines[0])
self.assertTrue('foo:FAILED is in RUNNING state, restarting' in lines[1])
self.assertTrue("Failed to stop process foo:FAILED: "
"<Fault 30: 'FAILED'>" in lines[2])
self.assertTrue('foo:FAILED restarted' in lines[3])
mailed = prog.mailed.split('\n')
self.assertEqual(len(mailed), 10)
self.assertEqual(mailed[0], 'To: chrism@plope.com')
Expand All @@ -272,12 +270,11 @@ def test_runforever_eager_error_on_process_start(self):
prog.stdin.seek(0)
prog.runforever(test=True)
lines = prog.stderr.getvalue().split('\n')
self.assertEqual(lines[0],
"Restarting selected processes ['SPAWN_ERROR']")
self.assertEqual(lines[1],
'foo:SPAWN_ERROR is in RUNNING state, restarting')
self.assertEqual(lines[2],
"Failed to start process foo:SPAWN_ERROR: <Fault 50: 'SPAWN_ERROR'>")
self.assertTrue("Restarting selected processes ['SPAWN_ERROR']" in lines[0])
self.assertTrue('foo:SPAWN_ERROR is in RUNNING state, restarting' in
lines[1])
self.assertTrue("Failed to start process foo:SPAWN_ERROR: "
"<Fault 50: 'SPAWN_ERROR'>" in lines[2])
mailed = prog.mailed.split('\n')
self.assertEqual(len(mailed), 9)
self.assertEqual(mailed[0], 'To: chrism@plope.com')
Expand All @@ -293,20 +290,18 @@ def test_runforever_eager_gcore(self):
prog.stdin.seek(0)
prog.runforever(test=True)
lines = prog.stderr.getvalue().split('\n')
self.assertEqual(lines[0],
("Restarting selected processes ['foo', 'bar', "
"'baz_01', 'notexisting']")
)
self.assertEqual(lines[1], 'gcore output for foo:')
self.assertTrue("Restarting selected processes ['foo', 'bar', "
"'baz_01', 'notexisting']" in lines[0])
self.assertTrue('gcore output for foo:' in lines[1])
self.assertEqual(lines[2], '')
self.assertEqual(lines[3], ' ')
self.assertEqual(lines[4], 'foo is in RUNNING state, restarting')
self.assertEqual(lines[5], 'foo restarted')
self.assertEqual(lines[6], 'bar not in RUNNING state, NOT restarting')
self.assertEqual(lines[7],
'baz:baz_01 not in RUNNING state, NOT restarting')
self.assertEqual(lines[8],
"Programs not restarted because they did not exist: ['notexisting']")
self.assertTrue('foo is in RUNNING state, restarting' in lines[4])
self.assertTrue('foo restarted' in lines[5])
self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[6])
self.assertTrue('baz:baz_01 not in RUNNING state, NOT restarting' in
lines[7])
self.assertTrue("Programs not restarted because they did not exist: "
"['notexisting']" in lines[8])
mailed = prog.mailed.split('\n')
self.assertEqual(len(mailed), 15)
self.assertEqual(mailed[0], 'To: chrism@plope.com')
Expand All @@ -333,12 +328,10 @@ def test_runforever_not_eager_running(self):
prog.stdin.seek(0)
prog.runforever(test=True)
lines = [x for x in prog.stderr.getvalue().split('\n') if x]
self.assertEqual(lines[0],
("Restarting selected processes ['foo', 'bar']")
)
self.assertEqual(lines[1], 'foo is in RUNNING state, restarting')
self.assertEqual(lines[2], 'foo restarted')
self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting')
self.assertTrue("Restarting selected processes ['foo', 'bar']" in lines[0])
self.assertTrue('foo is in RUNNING state, restarting' in lines[1])
self.assertTrue('foo restarted' in lines[2])
self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[3])
mailed = prog.mailed.split('\n')
self.assertEqual(len(mailed), 10)
self.assertEqual(mailed[0], 'To: chrism@plope.com')
Expand Down Expand Up @@ -368,12 +361,11 @@ def test_runforever_connrefused_error(self):
prog.stdin.seek(0)
prog.runforever(test=True)
lines = [x for x in prog.stderr.getvalue().split('\n') if x]
self.assertEqual(lines[0],
("Restarting selected processes ['foo', 'bar']")
)
self.assertEqual(lines[1], 'foo is in RUNNING state, restarting')
self.assertEqual(lines[2], 'foo restarted')
self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting')
self.assertTrue("Restarting selected processes ['foo', 'bar']"
in lines[0])
self.assertTrue('foo is in RUNNING state, restarting' in lines[1])
self.assertTrue('foo restarted' in lines[2])
self.assertTrue('bar not in RUNNING state, NOT restarting' in lines[3])
mailed = prog.mailed.split('\n')
self.assertEqual(len(mailed), 10)
self.assertEqual(mailed[0], 'To: chrism@plope.com')
Expand Down
65 changes: 32 additions & 33 deletions superlance/tests/memmon_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,13 @@ def test_runforever_tick_programs(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 8)
self.assertEqual(lines[0], 'Checking programs bar=0, baz_01=0, foo=0')
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertEqual(lines[2], 'Restarting foo:foo')
self.assertEqual(lines[3], 'RSS of bar:bar is 2265088')
self.assertEqual(lines[4], 'Restarting bar:bar')
self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088')
self.assertEqual(lines[6], 'Restarting baz:baz_01')
self.assertTrue('Checking programs bar=0, baz_01=0, foo=0' in lines[0])
self.assertTrue('RSS of foo:foo is 2264064' in lines[1])
self.assertTrue('Restarting foo:foo' in lines[2])
self.assertTrue('RSS of bar:bar is 2265088' in lines[3])
self.assertTrue('Restarting bar:bar' in lines[4])
self.assertTrue('RSS of baz:baz_01 is 2265088' in lines[5])
self.assertTrue('Restarting baz:baz_01' in lines[6])
self.assertEqual(lines[7], '')
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
Expand All @@ -80,9 +80,9 @@ def test_runforever_tick_groups(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 4)
self.assertEqual(lines[0], 'Checking groups foo=0')
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertEqual(lines[2], 'Restarting foo:foo')
self.assertTrue('Checking groups foo=0' in lines[0])
self.assertTrue('RSS of foo:foo is 2264064' in lines[1])
self.assertTrue('Restarting foo:foo' in lines[2])
self.assertEqual(lines[3], '')
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
Expand All @@ -102,13 +102,13 @@ def test_runforever_tick_any(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 8)
self.assertEqual(lines[0], 'Checking any=0')
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertEqual(lines[2], 'Restarting foo:foo')
self.assertEqual(lines[3], 'RSS of bar:bar is 2265088')
self.assertEqual(lines[4], 'Restarting bar:bar')
self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088')
self.assertEqual(lines[6], 'Restarting baz:baz_01')
self.assertTrue('Checking any=0' in lines[0])
self.assertTrue('RSS of foo:foo is 2264064' in lines[1])
self.assertTrue('Restarting foo:foo' in lines[2])
self.assertTrue('RSS of bar:bar is 2265088' in lines[3])
self.assertTrue('Restarting bar:bar' in lines[4])
self.assertTrue('RSS of baz:baz_01 is 2265088' in lines[5])
self.assertTrue('Restarting baz:baz_01' in lines[6])
self.assertEqual(lines[7], '')
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
Expand All @@ -123,12 +123,12 @@ def test_runforever_tick_programs_and_groups(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 7)
self.assertEqual(lines[0], 'Checking programs baz_01=0')
self.assertEqual(lines[1], 'Checking groups foo=0')
self.assertEqual(lines[2], 'RSS of foo:foo is 2264064')
self.assertEqual(lines[3], 'Restarting foo:foo')
self.assertEqual(lines[4], 'RSS of baz:baz_01 is 2265088')
self.assertEqual(lines[5], 'Restarting baz:baz_01')
self.assertTrue('Checking programs baz_01=0' in lines[0])
self.assertTrue('Checking groups foo=0' in lines[1])
self.assertTrue('RSS of foo:foo is 2264064' in lines[2])
self.assertTrue('Restarting foo:foo' in lines[3])
self.assertTrue('RSS of baz:baz_01 is 2265088' in lines[4])
self.assertTrue('Restarting baz:baz_01' in lines[5])
self.assertEqual(lines[6], '')
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
Expand All @@ -148,8 +148,8 @@ def test_runforever_tick_programs_norestart(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 3)
self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint)
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertTrue('Checking programs foo=%s' % maxint in lines[0])
self.assertTrue('RSS of foo:foo is 2264064' in lines[1])
self.assertEqual(lines[2], '')
self.assertEqual(memmon.mailed, False)

Expand All @@ -163,8 +163,8 @@ def test_stopprocess_fault_tick_programs_norestart(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 3)
self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint)
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertTrue('Checking programs foo=%s' % maxint in lines[0])
self.assertTrue('RSS of foo:foo is 2264064' in lines[1])
self.assertEqual(lines[2], '')
self.assertEqual(memmon.mailed, False)

Expand Down Expand Up @@ -192,17 +192,17 @@ def test_stopprocess_fails_to_stop(self):
self.assertRaises(xmlrpclib.Fault, memmon.runforever, True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 4)
self.assertEqual(lines[0], 'Checking programs BAD_NAME=%s' % 0)
self.assertEqual(lines[1], 'RSS of BAD_NAME:BAD_NAME is 2264064')
self.assertEqual(lines[2], 'Restarting BAD_NAME:BAD_NAME')
self.assertTrue(lines[3].startswith('Failed'))
self.assertTrue('Checking programs BAD_NAME=%s' % 0 in lines[0])
self.assertTrue('RSS of BAD_NAME:BAD_NAME is 2264064' in lines[1])
self.assertTrue('Restarting BAD_NAME:BAD_NAME' in lines[2])
self.assertTrue('Failed' in lines[3])
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
self.assertEqual(mailed[0], 'To: chrism@plope.com')
self.assertEqual(mailed[1],
'Subject: memmon: failed to stop process BAD_NAME:BAD_NAME, exiting')
self.assertEqual(mailed[2], '')
self.assertTrue(mailed[3].startswith('Failed'))
self.assertTrue('Failed' in mailed[3])

def test_subject_no_name(self):
"""set the name to None to check if subject formats to:
Expand Down Expand Up @@ -392,4 +392,3 @@ def test_argparser(self):

if __name__ == '__main__':
unittest.main()