Since the operation monitoring system is centrally managed by Zabbix, I wanted to notify Zabbix Server of the execution error of the AWS Lambda function, so I tried it.
Settings> Hosts> Create Host
--Hostname: lambda-trapper --Create new group: Lambda --Agent interface --IP address: 127.0.0.1 (default) --Connection method: IP address (default) --Port: 10050 (default) --Valid: On --Other items: Blank
[Settings]> [Hosts]> [Items]> [Create Item] of lambda-trapper in the host list
--Name: ErrorLog --Type: Zabbix Trapper --Key: errorlog --Data type: String --History retention period (days): 90 (default) --Use value mapping: None (default) --Create application: Logs --Automatic setting of host inventory fields: None (default) --Valid: On --Other items: Blank
[Settings]> [Hosts]> [Trigger]> [Create Trigger] of lambda-trapper in the host list
--Name: Lambda function execution error --Conditional expression: {lambda-trapper: errorlog.str (example)} = 0 and {lambda-trapper: errorlog.nodata (30)} = 0 --Continue to generate failure events: On --Severity: Severe disability --Valid: On --Other items: Blank
Add logging processing to the existing AWS Lambda function (the function you want to trap execution errors) as follows.
lambda_function.py
# (abridgement)
import logging
# (abridgement)
#Logger initialization
logger = logging.getLogger()
logger.setLevel(logging.INFO)
def lambda_handler(event, context):
try:
# (abridgement)
logger.info("Execution succeeded.")
except Exception, e:
logger.error(e)
raise(e) #Described when you want to record the Python stack trace in the CloudWatch log.
By adding the logging process, the execution result of the AWS Lambda function will be output to CloudWatch. Example of normal end log)
[INFO] 2017-03-01T08:18:31.149Z a7afba37-fe57-11e6-a428-371557c5f4e7 Execution succeeded.
Abnormal termination log example)
[ERROR] 2017-03-01T09:38:37.966Z d8c7ede3-fe62-11e6-b58a-7dce3a95f14a cannot concatenate 'str' and 'int' objects
Create a new AWS Lambda function for Zabbix submission kicked from CloudWatch. By the way, Zabbix Sender module is published in "here" I diverted the code.
cwl-to-zabbix
handler.py
import os
import json
from StringIO import StringIO
import base64
import gzip
from ZabbixSender import ZabbixSender
import logging
ZBX_SERVER = os.environ['ZBX_SERVER']
ZBX_HOST = os.environ['ZBX_HOST']
ZBX_ITEM = os.environ['ZBX_ITEM']
logger = logging.getLogger()
logger.setLevel(logging.INFO)
def sender(event, context):
log_b64 = event['awslogs']['data']
log_gz = StringIO(base64.b64decode(log_b64))
jsonData = json.load(gzip.GzipFile(fileobj=log_gz, mode='rb'))
status = jsonData['logEvents'][0]['extractedFields']['status']
message = jsonData['logEvents'][0]['extractedFields']['event']
timestamp = int(jsonData['logEvents'][0]['timestamp']) / 1000
func = os.path.basename(jsonData['logGroup'])
try:
if status == "ERROR":
errlog = func + ': ' + message
sender = ZabbixSender(ZBX_SERVER)
sender.add(ZBX_HOST, ZBX_ITEM, errlog, timestamp)
sender.send()
logger.info("Execution Succeeded.")
except Exception, e:
logger.error(e)
raise(e)
ZabbixSender.py
import socket
import struct
import time
import json
class ZabbixSender:
log = True
def __init__(self, host='127.0.0.1', port=10051):
self.address = (host, port)
self.data = []
def __log(self, log):
if self.log: print log
def __connect(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.sock.connect(self.address)
except:
raise Exception("Can't connect server.")
def __close(self):
self.sock.close()
def __pack(self, request):
string = json.dumps(request)
header = struct.pack('<4sBQ', 'ZBXD', 1, len(string))
return header + string
def __unpack(self, response):
header, version, length = struct.unpack('<4sBQ', response[:13])
(data, ) = struct.unpack('<%ds'%length, response[13:13+length])
return json.loads(data)
def __request(self, request):
self.__connect()
try:
self.sock.sendall(self.__pack(request))
except Exception as e:
raise Exception("Failed sending data.\nERROR: %s" % e)
response = ''
while True:
data = self.sock.recv(4096)
if not data:
break
response += data
self.__close()
return self.__unpack(response)
def __active_checks(self):
hosts = set()
for d in self.data:
hosts.add(d['host'])
for h in hosts:
request = {"request":"active checks", "host":h}
self.__log("[active check] %s" % h)
response = self.__request(request)
if not response['response'] == 'success': self.__log("[host not found] %s" % h)
def add(self, host, key, value, clock=None):
if clock is None: clock = int(time.time())
self.data.append({"host":host, "key":key, "value":value, "clock":clock})
def send(self):
if not self.data:
self.__log("Not found sender data, end without sending.")
return False
self.__active_checks()
request = {"request":"sender data", "data":self.data}
response = self.__request(request)
result = True if response['response'] == 'success' else False
if result:
for d in self.data:
self.__log("[send data] %s" % d)
self.__log("[send result] %s" % response['info'])
else:
raise Exception("Failed send data.")
return result
--ZBX_SERVER =
--Runtime: Python 2.7 --Handler: handler.sender --Role: Select an existing role --Existing roll: *
※ Create a role with the following access rights in advance and assign that role.
--Describe, Create, Delete authority to ENI (VPC connection is required to communicate with Zabbix server) --Create permission to LogGroup and LogStream of CloudWatch (for logging output from AWS Lambda function) --Put permission for CloudWatch LogEvent
--We do not consider the case where the AWS Lambda function for sending Zabbix itself is moss. -ZabbixSender Thanks to Gaido Dad for publishing the module. !! ――Serverless is interesting.
Recommended Posts