diff --git a/documentation/adding_custom_plugins.md b/documentation/adding_custom_plugins.md index 1afde825..614b479e 100644 --- a/documentation/adding_custom_plugins.md +++ b/documentation/adding_custom_plugins.md @@ -33,9 +33,9 @@ class PLUGIN_NAME(Plugin): # metrics collection interval in seconds Interval = PLUGIN INTERVAL - # Plugin type specifies which group new metrics will belong to. Plugin type can be one of 'pg', 'sys' or 'all: - # 'pg' configures PostgreSQL metrics - # 'sys' configures system metrics + # Plugin type specifies which group new metrics will belong to. Plugin type can be one of 'pgsql', 'system' or 'all: + # 'pgsql' configures PostgreSQL metrics + # 'system' configures system metrics # 'all' configures both PostgreSQL and system metrics AgentPluginType = 'PLUGIN TYPE' diff --git a/documentation/metrics.md b/documentation/metrics.md index 6d9428b9..daa94047 100644 --- a/documentation/metrics.md +++ b/documentation/metrics.md @@ -1690,7 +1690,7 @@ Default config: - *Files Need To Archive* is calculated as difference between current WAL number and last archived WAL number. You can find SQL-query that calculates this metric in plugin source [code](../mamonsu/plugins/pgsql/archive_command.py). + *Files Need To Archive* is calculated as difference between current WAL number and last archived WAL number. You can find SQL-query that calculates this metric in plugin source [code](../mamonsu/plugins/pgsql/archiver.py). 4. **Size Of Files Need To Archive** @@ -1723,7 +1723,7 @@ Default config: - *Size Of Files Need To Archive* is calculated as difference between current WAL number and last archived WAL number multiplied by `wal_segment_size`. You can find SQL-query that calculates this metric in plugin source [code](../mamonsu/plugins/pgsql/archive_command.py). + *Size Of Files Need To Archive* is calculated as difference between current WAL number and last archived WAL number multiplied by `wal_segment_size`. You can find SQL-query that calculates this metric in plugin source [code](../mamonsu/plugins/pgsql/archiver.py). ### Graphs diff --git a/documentation/tools.md b/documentation/tools.md index 3b828fd3..62bec9a4 100644 --- a/documentation/tools.md +++ b/documentation/tools.md @@ -128,9 +128,9 @@ Export metrics configuration for use with the native Zabbix agent. The optional **--plugin-type={pg | sys | all}** Specify the type of metrics to collect: - - 'pg' for PostgreSQL metrics. + - 'pgsql' for PostgreSQL metrics. - - 'sys' for system metrics. + - 'system' for system metrics. - 'all' for both PostgreSQL and system metrics. @@ -165,9 +165,9 @@ Export a template for use with the native Zabbix agent. The optional parameters **--plugin-type={pg | sys | all}** Specify the type of metrics to collect: - - 'pg' for PostgreSQL metrics. + - 'pgsql' for PostgreSQL metrics. - - 'sys' for system metrics. + - 'system' for system metrics. - 'all' for both PostgreSQL and system metrics. diff --git a/mamonsu/lib/const.py b/mamonsu/lib/const.py index 07d834c3..ca848644 100644 --- a/mamonsu/lib/const.py +++ b/mamonsu/lib/const.py @@ -28,6 +28,16 @@ class _template(object): 'EXTERNAL', 'ODBC', 'IPMI', 'SSH', 'TELNET', 'CALCULATED', 'JMX', 'SNMP_TRAP', 'DEPENDENT', 'HTTP_AGENT', 'SNMP_AGENT', 'ITEM_TYPE_SCRIPT']) TYPE = _type(0, 2, 3, 5, 7, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21) + # type of graph item drawing style + _drawtype = namedtuple( + 'drawtype', ['LINE', 'FILLED_REGION', 'BOLD_LINE', 'DOT', 'DASHED_LINE', 'GRADIENT_LINE'] + ) + DRAWTYPE = _drawtype(0, 1, 2, 3, 4, 5) + # side of graph item on graph + _yaxisside = namedtuple( + 'yaxisside', ['LEFT', 'RIGHT'] + ) + YAXISSIDE = _yaxisside(0, 1) class _api(object): diff --git a/mamonsu/lib/plugin.py b/mamonsu/lib/plugin.py index 48e7dfa2..841cc53e 100644 --- a/mamonsu/lib/plugin.py +++ b/mamonsu/lib/plugin.py @@ -52,6 +52,8 @@ class Plugin(object): VALUE_TYPE = Template.VALUE_TYPE UNITS = Template.UNITS TYPE = Template.TYPE + DRAWTYPE = Template.DRAWTYPE + YAXISSIDE = Template.YAXISSIDE DELTA_SPEED = Template.DELTA.speed_per_second DELTA_CHANGE = Template.DELTA.simple_change @@ -198,7 +200,7 @@ def right_type(self, key, var="", var_discovery=""): else: new_key = key.format('[{0}{1}]'.format(var, var_discovery[:-1])) else: - if self.AgentPluginType == 'sys': + if self.AgentPluginType == 'system': if var_discovery != "": if var == "": new_key = key.format('{0}[{1}]'.format(var, var_discovery)) @@ -215,3 +217,12 @@ def right_type(self, key, var="", var_discovery=""): else: new_key = key.format('.{0}[{1}]'.format(var, var_discovery + self.Macros[self.Type])) return new_key + + @staticmethod + def generate_prefixes(plugin_type, plugin_name): + return { + "key_prefix": "{0}.{1}".format(plugin_type.lower(), "_".join(plugin_name.split()).lower()), + "name_prefix": "{0} {1}: ".format("PosgreSQL " if plugin_type == "pgsql" else + "System" if plugin_type == "system" else "", + " ".join([(word[0].upper() + word[1:].lower()) for word in plugin_name.split()])) + } diff --git a/mamonsu/lib/runner.py b/mamonsu/lib/runner.py index 197c90ce..40fa8ed8 100644 --- a/mamonsu/lib/runner.py +++ b/mamonsu/lib/runner.py @@ -94,7 +94,7 @@ def quit_handler(_signo=None, _stack_frame=None): if klass.__name__ != "PgWaitSampling" and klass.__name__ != "Cfs": plugins.append(klass(cfg)) args.plugin_type = correct_plugin_type(args.plugin_type) - if args.plugin_type == 'pg' or args.plugin_type == 'sys' or args.plugin_type == 'all': + if args.plugin_type == 'pgsql' or args.plugin_type == 'system' or args.plugin_type == 'all': template = GetKeys() # write conf file try: @@ -163,7 +163,7 @@ def quit_handler(_signo=None, _stack_frame=None): commands.append('postgrespro_agent.xml') plugins = [] args.plugin_type = correct_plugin_type(args.plugin_type) - if args.plugin_type == 'pg' or args.plugin_type == 'sys' or args.plugin_type == 'all': + if args.plugin_type == 'pgsql' or args.plugin_type == 'system' or args.plugin_type == 'all': for klass in Plugin.only_child_subclasses(): if klass.__name__ != "PgWaitSampling" and klass.__name__ != "Cfs": # check if plugin is for EE plugins.append(klass(cfg)) @@ -244,7 +244,7 @@ def define_pg_version(version_args): def correct_plugin_type(plugin_type): types = plugin_type.split(',') # if number of plugin types is more than 1 and plugin types are valid => plugin type should be 'all' - valid_plugin_types = ('pg', 'sys', 'all') + valid_plugin_types = ('pgsql', 'system', 'all') if len(types) == 2 or len(types) == 3: # check if any plugin types is equal if is_any_equal(types): diff --git a/mamonsu/plugins/common/health.py b/mamonsu/plugins/common/health.py index 2013ed2f..b3da6b26 100644 --- a/mamonsu/plugins/common/health.py +++ b/mamonsu/plugins/common/health.py @@ -7,7 +7,7 @@ class Health(Plugin): - AgentPluginType = "sys" + AgentPluginType = "system" DEFAULT_CONFIG = { "max_memory_usage": str(40 * 1024 * 1024) diff --git a/mamonsu/plugins/pgsql/__init__.py b/mamonsu/plugins/pgsql/__init__.py index 2dcf8ef7..813d3314 100644 --- a/mamonsu/plugins/pgsql/__init__.py +++ b/mamonsu/plugins/pgsql/__init__.py @@ -5,7 +5,7 @@ __all__ += ['statements', 'pg_buffercache', 'wait_sampling'] __all__ += ['checkpoint', 'oldest', 'pg_locks'] __all__ += ['cfs'] -__all__ += ['archive_command'] +__all__ += ['archiver'] __all__ += ['prepared_transaction'] __all__ += ['relations_size'] diff --git a/mamonsu/plugins/pgsql/archive_command.py b/mamonsu/plugins/pgsql/archive_command.py deleted file mode 100644 index 04c4515c..00000000 --- a/mamonsu/plugins/pgsql/archive_command.py +++ /dev/null @@ -1,207 +0,0 @@ -# -*- coding: utf-8 -*- - -from mamonsu.plugins.pgsql.plugin import PgsqlPlugin as Plugin -from distutils.version import LooseVersion -from .pool import Pooler -from mamonsu.lib.zbx_template import ZbxTemplate -import re - - -class ArchiveCommand(Plugin): - AgentPluginType = "pg" - DEFAULT_CONFIG = { - "max_count_files": str(2) - } - Interval = 60 - - # if streaming replication is on, archive queue length and size will always be 0 for replicas - query_agent_count_files = """ - WITH values AS ( - SELECT - 4096/(ceil(pg_settings.setting::numeric/1024/1024)) AS segment_parts_count, - setting::bigint AS segment_size, - ('x' || substring(pg_stat_archiver.last_archived_wal from 9 for 8))::bit(32)::int AS last_wal_div, - ('x' || substring(pg_stat_archiver.last_archived_wal from 17 for 8))::bit(32)::int AS last_wal_mod, - CASE WHEN pg_is_in_recovery() THEN NULL ELSE - ('x' || substring(pg_{1}_name(pg_current_{0}()) from 9 for 8))::bit(32)::int END AS current_wal_div, - CASE WHEN pg_is_in_recovery() THEN NULL ELSE - ('x' || substring(pg_{1}_name(pg_current_{0}()) from 17 for 8))::bit(32)::int END AS current_wal_mod - FROM pg_settings, pg_stat_archiver - WHERE pg_settings.name = 'wal_segment_size') - SELECT greatest(coalesce((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1, 0), 0)::bigint AS count_files - FROM values; - """ - query_agent_size_files = """ - WITH values AS ( - SELECT - 4096/(ceil(pg_settings.setting::numeric/1024/1024)) AS segment_parts_count, - setting::bigint AS segment_size, - ('x' || substring(pg_stat_archiver.last_archived_wal from 9 for 8))::bit(32)::int AS last_wal_div, - ('x' || substring(pg_stat_archiver.last_archived_wal from 17 for 8))::bit(32)::int AS last_wal_mod, - CASE WHEN pg_is_in_recovery() THEN NULL ELSE - ('x' || substring(pg_{1}_name(pg_current_{0}()) from 9 for 8))::bit(32)::int END AS current_wal_div, - CASE WHEN pg_is_in_recovery() THEN NULL ELSE - ('x' || substring(pg_{1}_name(pg_current_{0}()) from 17 for 8))::bit(32)::int END AS current_wal_mod - FROM pg_settings, pg_stat_archiver - WHERE pg_settings.name = 'wal_segment_size') - greatest(coalesce(((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1) * segment_size, 0), 0)::bigint AS size_files - FROM values; - """ - - query_agent_archived_count = """ - SELECT archived_count FROM pg_stat_archiver; - """ - query_agent_failed_count = """ - SELECT failed_count FROM pg_stat_archiver; - """ - key = "pgsql.archive_command{0}" - name = "PostgreSQL archive command {0}" - Items = [ - # key, desc, color, side, graph, delta, units - ("count_files_to_archive", "Files in archive_status Need to Archive Count", "9C8A4E", 0, 1, Plugin.DELTA.as_is, Plugin.UNITS.none), - ("size_files_to_archive", "Files Need to Archive Size", "793F5D", 0, 0, Plugin.DELTA.as_is, Plugin.UNITS.bytes), - ("archived_files", "Archived Files Count", "578159", 0, 1, Plugin.DELTA.simple_change, Plugin.UNITS.none), - ("failed_trying_to_archive", "Attempts to Archive Files Count", "E57862", 0, 1, Plugin.DELTA.simple_change, Plugin.UNITS.none), - ] - old_archived_count = None - old_failed_count = None - - def run(self, zbx): - query_queue = """ - WITH values AS ( - SELECT - 4096/(ceil(pg_settings.setting::numeric/1024/1024)) AS segment_parts_count, - setting::bigint AS segment_size, - ('x' || substring(pg_stat_archiver.last_archived_wal from 9 for 8))::bit(32)::int AS last_wal_div, - ('x' || substring(pg_stat_archiver.last_archived_wal from 17 for 8))::bit(32)::int AS last_wal_mod, - CASE WHEN pg_is_in_recovery() THEN NULL ELSE - ('x' || substring(pg_{1}_name(pg_current_{0}()) from 9 for 8))::bit(32)::int END AS current_wal_div, - CASE WHEN pg_is_in_recovery() THEN NULL ELSE - ('x' || substring(pg_{1}_name(pg_current_{0}()) from 17 for 8))::bit(32)::int END AS current_wal_mod - FROM pg_settings, pg_stat_archiver - WHERE pg_settings.name = 'wal_segment_size') - SELECT greatest(coalesce((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1, 0), 0)::bigint AS count_files, - greatest(coalesce(((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1) * segment_size, 0), 0)::bigint AS size_files - FROM values; - """ - - self.disable_and_exit_if_archive_mode_is_not_on() - - if Pooler.is_bootstraped() and Pooler.bootstrap_version_greater("2.3.4"): - result_stats = Pooler.query(""" - SELECT * - FROM mamonsu.archive_stat(); - """) - else: - result_stats = Pooler.query(""" - SELECT archived_count, - failed_count - FROM pg_stat_archiver; - """) - current_archived_count = result_stats[0][0] - current_failed_count = result_stats[0][1] - if self.old_archived_count is not None: - archived_count = current_archived_count - self.old_archived_count - zbx.send("pgsql.archive_command[{0}]".format(self.Items[2][0]), archived_count) - if self.old_failed_count is not None: - failed_count = current_failed_count - self.old_failed_count - zbx.send("pgsql.archive_command[{0}]".format(self.Items[3][0]), failed_count) - self.old_archived_count = current_archived_count - self.old_failed_count = current_failed_count - - # check the last WAL file name to avoid XXX.history, XXX.partial, etc. - wal_exists = bool(re.search(r'^[0-9A-Z]{24}$', str( - Pooler.query(""" - SELECT pg_stat_archiver.last_archived_wal - FROM pg_stat_archiver; - """)[0][0]))) - if wal_exists: - if Pooler.is_bootstraped() and Pooler.bootstrap_version_greater("2.3.4"): - result_queue = Pooler.query(""" - SELECT * - FROM mamonsu.archive_command_files(); - """) - else: - if Pooler.server_version_greater("10.0"): - result_queue = Pooler.query(query_queue.format("wal_lsn", "walfile")) - else: - result_queue = Pooler.query(query_queue.format("xlog_location", "xlogfile")) - zbx.send("pgsql.archive_command[{0}]".format(self.Items[0][0]), result_queue[0][0]) - zbx.send("pgsql.archive_command[{0}]".format(self.Items[1][0]), result_queue[0][1]) - - def items(self, template, dashboard=False): - result = "" - for item in self.Items: - result += template.item({ - "key": self.right_type(self.key, item[0]), - "name": "PostgreSQL Archiver: {0}".format(self.name.format(item[1])), - "value_type": self.VALUE_TYPE.numeric_unsigned, - "delay": self.plugin_config("interval"), - "delta": item[5], - "units": item[6] - }) - if not dashboard: - return result - else: - return [{ - "dashboard": {"name": self.right_type(self.key, self.Items[1][0]), - "page": ZbxTemplate.dashboard_page_wal["name"], - "size": ZbxTemplate.dashboard_widget_size_medium, - "position": 3} - }, - { - "dashboard": {"name": self.right_type(self.key, self.Items[2][0]), - "page": ZbxTemplate.dashboard_page_wal["name"], - "size": ZbxTemplate.dashboard_widget_size_medium, - "position": 4} - }] - - def graphs(self, template, dashboard=False): - graph = [] - result = "" - for item in self.Items: - if item[4] == 1: - graph.append({ - "key": self.right_type(self.key, item[0]), "color": item[2], "yaxisside": item[3], "drawtype": 2 - }) - result += template.graph({ - "name": self.name.format("") + " archive status ", - "items": graph - }) - if not dashboard: - return result - else: - return [{ - "dashboard": {"name": self.name.format("") + " archive status ", - "page": ZbxTemplate.dashboard_page_wal["name"], - "size": ZbxTemplate.dashboard_widget_size_medium, - "position": 1} - }] - - def triggers(self, template, dashboard=False): - return template.trigger({ - "name": "PostgreSQL count files in ./archive_status on {HOSTNAME} more than 2", - "expression": "{#TEMPLATE:" + self.right_type(self.key, - self.Items[0][0]) + ".last()}>" + self.plugin_config( - "max_count_files") - }) - - def keys_and_queries(self, template_zabbix): - result = [] - if LooseVersion(self.VersionPG) >= LooseVersion("10"): - result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[0][0]), - self.query_agent_count_files.format("wal_lsn", "walfile"))) - result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[1][0]), - self.query_agent_size_files.format("wal_lsn", "walfile"))) - else: - result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[0][0]), - self.query_agent_count_files.format("xlog_location", - "xlogfile"))) - result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[1][0]), - self.query_agent_size_files.format("xlog_location", - "xlogfile"))) - result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[2][0]), - self.query_agent_archived_count)) - result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[3][0]), - self.query_agent_failed_count)) - return template_zabbix.key_and_query(result) diff --git a/mamonsu/plugins/pgsql/archiver.py b/mamonsu/plugins/pgsql/archiver.py new file mode 100644 index 00000000..65c2499e --- /dev/null +++ b/mamonsu/plugins/pgsql/archiver.py @@ -0,0 +1,245 @@ +# -*- coding: utf-8 -*- + +from mamonsu.plugins.pgsql.plugin import PgsqlPlugin as Plugin +from distutils.version import LooseVersion +from .pool import Pooler +from mamonsu.lib.zbx_template import ZbxTemplate +import re + + +class Archiver(Plugin): + + AgentPluginType = "pgsql" + + DEFAULT_CONFIG = { + "max_count_files": str(2) + } + + Interval = 60 + + # if streaming replication is on, archive queue length and size will always be 0 for replicas + items_queue = { + "count_files_to_archive": { + "name": "Files Need to Archive Count", + "value_type": Plugin.VALUE_TYPE.numeric_unsigned, + "units": Plugin.UNITS.none, + "delta": Plugin.DELTA.as_is, + "color": "9C8A4E", + "drawtype": Plugin.DRAWTYPE.BOLD_LINE, + "yaxisside": Plugin.YAXISSIDE.LEFT, + "query": { + "standard": """ + WITH values AS ( + SELECT + 4096/(ceil(pg_settings.setting::numeric/1024/1024)) AS segment_parts_count, + setting::bigint AS segment_size, + ('x' || substring(pg_stat_archiver.last_archived_wal from 9 for 8))::bit(32)::int AS last_wal_div, + ('x' || substring(pg_stat_archiver.last_archived_wal from 17 for 8))::bit(32)::int AS last_wal_mod, + CASE WHEN pg_is_in_recovery() THEN NULL ELSE + ('x' || substring(pg_{1}_name(pg_current_{0}()) from 9 for 8))::bit(32)::int END AS current_wal_div, + CASE WHEN pg_is_in_recovery() THEN NULL ELSE + ('x' || substring(pg_{1}_name(pg_current_{0}()) from 17 for 8))::bit(32)::int END AS current_wal_mod + FROM pg_settings, pg_stat_archiver + WHERE pg_settings.name = 'wal_segment_size') + SELECT greatest(coalesce((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1, 0), 0)::bigint AS files_count + FROM values; + """, + "bootstrap": """ + SELECT files_count FROM mamonsu.archive_command_files(); + """ + } + }, + "size_files_to_archive": { + "name": "Files Need to Archive Size", + "value_type": Plugin.VALUE_TYPE.numeric_unsigned, + "units": Plugin.UNITS.bytes, + "delta": Plugin.DELTA.as_is, + "color": "793F5D", + "drawtype": Plugin.DRAWTYPE.BOLD_LINE, + "yaxisside": Plugin.YAXISSIDE.RIGHT, + "query": { + "standard": """ + WITH values AS ( + SELECT + 4096/(ceil(pg_settings.setting::numeric/1024/1024)) AS segment_parts_count, + setting::bigint AS segment_size, + ('x' || substring(pg_stat_archiver.last_archived_wal from 9 for 8))::bit(32)::int AS last_wal_div, + ('x' || substring(pg_stat_archiver.last_archived_wal from 17 for 8))::bit(32)::int AS last_wal_mod, + CASE WHEN pg_is_in_recovery() THEN NULL ELSE + ('x' || substring(pg_{1}_name(pg_current_{0}()) from 9 for 8))::bit(32)::int END AS current_wal_div, + CASE WHEN pg_is_in_recovery() THEN NULL ELSE + ('x' || substring(pg_{1}_name(pg_current_{0}()) from 17 for 8))::bit(32)::int END AS current_wal_mod + FROM pg_settings, pg_stat_archiver + WHERE pg_settings.name = 'wal_segment_size') + greatest(coalesce(((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1) * segment_size, 0), 0)::bigint AS files_size + FROM values; + """, + "bootstrap": """ + SELECT files_size FROM mamonsu.archive_command_files(); + """ + } + } + } + + items_status = { + "archived_files": { + "name": "Archived Files Count (in {0} seconds)".format(Interval), + "value_type": Plugin.VALUE_TYPE.numeric_unsigned, + "units": Plugin.UNITS.none, + "delta": Plugin.DELTA.simple_change, + "color": "578159", + "drawtype": Plugin.DRAWTYPE.BOLD_LINE, + "yaxisside": Plugin.YAXISSIDE.LEFT, + "query": { + "standard": """ + SELECT archived_count FROM pg_stat_archiver; + """, + "bootstrap": """ + SELECT archived_count FROM mamonsu.archive_stat(); + """ + } + }, + "failed_trying_to_archive": { + "name": "Failed Attempts to Archive Files Count (in {0} seconds)".format(Interval), + "value_type": Plugin.VALUE_TYPE.numeric_unsigned, + "units": Plugin.UNITS.none, + "delta": Plugin.DELTA.simple_change, + "color": "E57862", + "drawtype": Plugin.DRAWTYPE.BOLD_LINE, + "yaxisside": Plugin.YAXISSIDE.LEFT, + "query": { + "standard": """ + SELECT failed_count FROM pg_stat_archiver; + """, + "bootstrap": """ + SELECT failed_count FROM mamonsu.archive_stat(); + """ + } + } + } + + plugin_graphs = { + "queue": { + "name": "Queue", + "items": items_queue + }, + "status": { + "name": "Status", + "items": items_status + } + } + + prefixes = Plugin.generate_prefixes(plugin_type=AgentPluginType, plugin_name="Archiver") + + old_archived_count = None + old_failed_count = None + + def run(self, zbx): + + self.disable_and_exit_if_archive_mode_is_not_on() + + if Pooler.is_bootstraped() and Pooler.bootstrap_version_greater("2.3.4"): + query_type = "bootstrap" + else: + query_type = "standard" + + current_archived_count = Pooler.query(self.items_status["archived_files"]["query"][query_type])[0][0] + current_failed_count = Pooler.query(self.items_status["failed_trying_to_archive"]["query"][query_type])[0][0] + if self.old_archived_count is not None: + archived_count = current_archived_count - self.old_archived_count + zbx.send("{0}[{1}]".format(self.prefixes["key_prefix"], "archived_files"), archived_count) + if self.old_failed_count is not None: + failed_count = current_failed_count - self.old_failed_count + zbx.send("{0}[{1}]".format(self.prefixes["key_prefix"], "failed_trying_to_archive"), failed_count) + self.old_archived_count = current_archived_count + self.old_failed_count = current_failed_count + + # check the last WAL file name to avoid XXX.history, XXX.partial, etc. + wal_exists = bool(re.search(r'^[0-9A-Z]{24}$', str( + Pooler.query(""" + SELECT pg_stat_archiver.last_archived_wal + FROM pg_stat_archiver; + """)[0][0]))) + if wal_exists: + queue_length = Pooler.query(self.items_queue["count_files_to_archive"]["query"][query_type] + .format("wal_lsn", "walfile") if Pooler.server_version_greater("10.0") else + self.items_queue["count_files_to_archive"]["query"][query_type] + .format("xlog_location", "xlogfile"))[0][0] + queue_size = Pooler.query(self.items_queue["size_files_to_archive"]["query"][query_type] + .format("wal_lsn", "walfile") if Pooler.server_version_greater("10.0") else + self.items_queue["size_files_to_archive"]["query"][query_type] + .format("xlog_location", "xlogfile"))[0][0] + zbx.send("{0}[{1}]".format(self.prefixes["key_prefix"], "count_files_to_archive"), queue_length) + zbx.send("{0}[{1}]".format(self.prefixes["key_prefix"], "size_files_to_archive"), queue_size) + + def items(self, template, dashboard=False): + result = "" + for item, properties in {**self.items_queue, **self.items_status}.items(): + result += template.item({ + "key": self.right_type(self.prefixes["key_prefix"] + "{0}", item), + "name": self.prefixes["name_prefix"] + "{0}".format(properties["name"]), + "value_type": properties["value_type"], + "delay": self.plugin_config("interval"), + "units": properties["units"], + "delta": properties["delta"] + }) + if not dashboard: + return result + else: + return [{ + "dashboard": {"name": self.right_type(self.prefixes["key_prefix"] + "{0}", "size_files_to_archive"), + "page": ZbxTemplate.dashboard_page_wal["name"], + "size": ZbxTemplate.dashboard_widget_size_medium, + "position": 3} + }, + { + "dashboard": {"name": self.right_type(self.prefixes["key_prefix"] + "{0}", "archived_files"), + "page": ZbxTemplate.dashboard_page_wal["name"], + "size": ZbxTemplate.dashboard_widget_size_medium, + "position": 4} + }] + + def graphs(self, template, dashboard=False): + result = "" + items = [] + for graph, graph_properties in self.plugin_graphs.items(): + for item, item_properties in graph_properties["items"].items(): + items.append({ + "key": self.right_type(self.prefixes["key_prefix"] + "{0}", item), + "color": item_properties["color"], + "drawtype": item_properties["drawtype"], + "yaxisside": item_properties["yaxisside"] + }) + result += template.graph({ + "name": self.prefixes["name_prefix"] + "{0}".format(graph_properties["name"]), + "items": items + }) + items.clear() + if not dashboard: + return result + else: + return [{ + "dashboard": {"name": self.prefixes["name_prefix"] + "{0}".format(self.plugin_graphs["queue"]["name"]), + "page": ZbxTemplate.dashboard_page_wal["name"], + "size": ZbxTemplate.dashboard_widget_size_medium, + "position": 1} + }] + + def triggers(self, template, dashboard=False): + return template.trigger({ + "name": "PostgreSQL Files Need to Archive Queue on {HOSTNAME} more than 2", + "expression": "{#TEMPLATE:" + self.right_type(self.prefixes["key_prefix"] + "{0}", "count_files_to_archive") + + ".last()}>" + self.plugin_config("max_count_files") + }) + + def keys_and_queries(self, template_zabbix): + result = [] + for item, properties in self.items_queue.items(): + result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.prefixes["key_prefix"] + "." + item, + properties["query"]["standard"].format("wal_lsn", "walfile") + if LooseVersion(self.VersionPG) >= LooseVersion("10") else + properties["query"]["standard"].format("xlog_location", "xlogfile"))) + for item, properties in self.items_status.items(): + result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.prefixes["key_prefix"] + "." + item, + properties["query"]["standard"])) + return template_zabbix.key_and_query(result) diff --git a/mamonsu/plugins/pgsql/bgwriter.py b/mamonsu/plugins/pgsql/bgwriter.py index 576440e2..d80512b3 100644 --- a/mamonsu/plugins/pgsql/bgwriter.py +++ b/mamonsu/plugins/pgsql/bgwriter.py @@ -6,7 +6,7 @@ class BgWriter(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" key = "pgsql.bgwriter{0}" query = """ SELECT {0} diff --git a/mamonsu/plugins/pgsql/cfs.py b/mamonsu/plugins/pgsql/cfs.py index 54e281de..676e4eba 100644 --- a/mamonsu/plugins/pgsql/cfs.py +++ b/mamonsu/plugins/pgsql/cfs.py @@ -5,7 +5,7 @@ class Cfs(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" ratioInterval, ratioCounter = 10, 0 timeRatioInterval = ratioInterval * 60 diff --git a/mamonsu/plugins/pgsql/checkpoint.py b/mamonsu/plugins/pgsql/checkpoint.py index 5ec6906e..5aee2c04 100644 --- a/mamonsu/plugins/pgsql/checkpoint.py +++ b/mamonsu/plugins/pgsql/checkpoint.py @@ -6,7 +6,7 @@ class Checkpoint(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" Interval = 60 * 5 query = """ diff --git a/mamonsu/plugins/pgsql/connections.py b/mamonsu/plugins/pgsql/connections.py index 986373fd..2f29d368 100644 --- a/mamonsu/plugins/pgsql/connections.py +++ b/mamonsu/plugins/pgsql/connections.py @@ -7,7 +7,7 @@ class Connections(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" # (state, key, name, graph) DEFAULT_CONFIG = { "percent_connections_tr": str(90) diff --git a/mamonsu/plugins/pgsql/databases.py b/mamonsu/plugins/pgsql/databases.py index c99a1f8d..25f62bfb 100644 --- a/mamonsu/plugins/pgsql/databases.py +++ b/mamonsu/plugins/pgsql/databases.py @@ -8,7 +8,7 @@ class Databases(Plugin): Interval = 60 * 5 - AgentPluginType = "pg" + AgentPluginType = "pgsql" # queries to form sql files query_bloating_tables = """ SELECT count(*) diff --git a/mamonsu/plugins/pgsql/health.py b/mamonsu/plugins/pgsql/health.py index 49253345..2a98e279 100644 --- a/mamonsu/plugins/pgsql/health.py +++ b/mamonsu/plugins/pgsql/health.py @@ -7,7 +7,7 @@ class PgHealth(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" DEFAULT_CONFIG = { "uptime": str(60 * 10), "cache": str(80) diff --git a/mamonsu/plugins/pgsql/instance.py b/mamonsu/plugins/pgsql/instance.py index 69a538ae..1c774ed2 100644 --- a/mamonsu/plugins/pgsql/instance.py +++ b/mamonsu/plugins/pgsql/instance.py @@ -7,7 +7,7 @@ class Instance(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" query_agent = """ SELECT sum({0}) AS {0} FROM pg_catalog.pg_stat_database; diff --git a/mamonsu/plugins/pgsql/oldest.py b/mamonsu/plugins/pgsql/oldest.py index 12b7ce4d..efc57d5b 100644 --- a/mamonsu/plugins/pgsql/oldest.py +++ b/mamonsu/plugins/pgsql/oldest.py @@ -6,7 +6,7 @@ class Oldest(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" key = "pgsql.oldest{0}" OldestXidSql = """ SELECT greatest(max(age(backend_xmin)), diff --git a/mamonsu/plugins/pgsql/pg_buffercache.py b/mamonsu/plugins/pgsql/pg_buffercache.py index 45c54bfc..4240d281 100644 --- a/mamonsu/plugins/pgsql/pg_buffercache.py +++ b/mamonsu/plugins/pgsql/pg_buffercache.py @@ -6,7 +6,7 @@ class PgBufferCache(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" key = "pgsql.buffers{0}" query_agent_size = """ SELECT sum(1) * (current_setting('block_size')::int8) AS size diff --git a/mamonsu/plugins/pgsql/pg_locks.py b/mamonsu/plugins/pgsql/pg_locks.py index aa37b05c..7dfe33eb 100644 --- a/mamonsu/plugins/pgsql/pg_locks.py +++ b/mamonsu/plugins/pgsql/pg_locks.py @@ -6,7 +6,7 @@ class PgLocks(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" query = """ SELECT lower(mode), count(mode) diff --git a/mamonsu/plugins/pgsql/statements.py b/mamonsu/plugins/pgsql/statements.py index e229587d..e2f99e2e 100644 --- a/mamonsu/plugins/pgsql/statements.py +++ b/mamonsu/plugins/pgsql/statements.py @@ -6,7 +6,7 @@ class Statements(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" query = { "pg_stat_statements": """ diff --git a/mamonsu/plugins/pgsql/wait_sampling.py b/mamonsu/plugins/pgsql/wait_sampling.py index 3c2fb97b..acc550e1 100644 --- a/mamonsu/plugins/pgsql/wait_sampling.py +++ b/mamonsu/plugins/pgsql/wait_sampling.py @@ -5,7 +5,7 @@ class WaitSampling(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" AllLockItems = [ # (sql_key, zbx_key, name, color) diff --git a/mamonsu/plugins/pgsql/wal.py b/mamonsu/plugins/pgsql/wal.py index a22fe15c..e6f729c9 100644 --- a/mamonsu/plugins/pgsql/wal.py +++ b/mamonsu/plugins/pgsql/wal.py @@ -9,7 +9,7 @@ class Wal(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" DEFAULT_CONFIG = { "lag_more_than_in_sec": str(60 * 5) } diff --git a/mamonsu/plugins/system/linux/disk_sizes.py b/mamonsu/plugins/system/linux/disk_sizes.py index fc79fcc6..9671434b 100644 --- a/mamonsu/plugins/system/linux/disk_sizes.py +++ b/mamonsu/plugins/system/linux/disk_sizes.py @@ -3,7 +3,7 @@ class DiskSizes(Plugin): - AgentPluginType = "sys" + AgentPluginType = "system" query_agent_discovery = "/disk_sizes.sh -j MOUNTPOINT" query_agent_used = "df $1 | awk 'NR == 2 {print $$3 * 1024}'" diff --git a/mamonsu/plugins/system/linux/disk_stats.py b/mamonsu/plugins/system/linux/disk_stats.py index dfc3b72b..882d0d5a 100644 --- a/mamonsu/plugins/system/linux/disk_stats.py +++ b/mamonsu/plugins/system/linux/disk_stats.py @@ -5,7 +5,7 @@ class DiskStats(Plugin): # todo yaxis right 100% - AgentPluginType = "sys" + AgentPluginType = "system" query_agent_discovery = "/disk_stats.sh -j BLOCKDEVICE" agent_query_read_op = "expr `grep -w '$1' /proc/diskstats | awk '{print $$4}'`" diff --git a/mamonsu/plugins/system/linux/la.py b/mamonsu/plugins/system/linux/la.py index 0c5d4b21..1ad18bae 100644 --- a/mamonsu/plugins/system/linux/la.py +++ b/mamonsu/plugins/system/linux/la.py @@ -2,7 +2,7 @@ class La(Plugin): - AgentPluginType = "sys" + AgentPluginType = "system" query_agent = "cat /proc/loadavg | awk '{ print $1 }'" key = "system.la{0}" diff --git a/mamonsu/plugins/system/linux/memory.py b/mamonsu/plugins/system/linux/memory.py index 51cef292..82c31b09 100644 --- a/mamonsu/plugins/system/linux/memory.py +++ b/mamonsu/plugins/system/linux/memory.py @@ -3,7 +3,7 @@ class Memory(Plugin): - AgentPluginType = "sys" + AgentPluginType = "system" query_agent = "cat /proc/meminfo | awk '/^{0}\:/ " query_agent_used = "MemTotal=$(cat /proc/meminfo | awk '/MemTotal\:/ { print $2 }'); " \ diff --git a/mamonsu/plugins/system/linux/net.py b/mamonsu/plugins/system/linux/net.py index baf82dd9..51e192e4 100644 --- a/mamonsu/plugins/system/linux/net.py +++ b/mamonsu/plugins/system/linux/net.py @@ -2,7 +2,7 @@ class Net(Plugin): - AgentPluginType = "sys" + AgentPluginType = "system" query_agent_discovery = "/net.sh -j NETDEVICE" query_agent = "expr `grep -Ei '$1' /proc/net/dev | awk '{print $$" diff --git a/mamonsu/plugins/system/linux/open_files.py b/mamonsu/plugins/system/linux/open_files.py index e7c91b6c..49f6f4ad 100644 --- a/mamonsu/plugins/system/linux/open_files.py +++ b/mamonsu/plugins/system/linux/open_files.py @@ -2,7 +2,7 @@ class OpenFiles(Plugin): - AgentPluginType = "sys" + AgentPluginType = "system" query_agent = "cat /proc/sys/fs/file-nr | awk '{ print $1 }'" key = "system.open_files{0}" diff --git a/mamonsu/plugins/system/linux/pg_probackup.py b/mamonsu/plugins/system/linux/pg_probackup.py index a5e81ffe..e5e6b073 100644 --- a/mamonsu/plugins/system/linux/pg_probackup.py +++ b/mamonsu/plugins/system/linux/pg_probackup.py @@ -6,7 +6,7 @@ class PgProbackup(Plugin): - AgentPluginType = "pg" + AgentPluginType = "pgsql" os_walk_error = None block_size = 4096 diff --git a/mamonsu/plugins/system/linux/proc_stat.py b/mamonsu/plugins/system/linux/proc_stat.py index c8c3e798..ddd78784 100644 --- a/mamonsu/plugins/system/linux/proc_stat.py +++ b/mamonsu/plugins/system/linux/proc_stat.py @@ -5,7 +5,7 @@ class ProcStat(Plugin): - AgentPluginType = "sys" + AgentPluginType = "system" query_agent = "cat /proc/stat" query_agent_procs = ["cat /proc/stat | awk '/procs_running/ { print $2 }'", diff --git a/mamonsu/plugins/system/linux/uptime.py b/mamonsu/plugins/system/linux/uptime.py index 94d07867..809c6dde 100644 --- a/mamonsu/plugins/system/linux/uptime.py +++ b/mamonsu/plugins/system/linux/uptime.py @@ -2,7 +2,7 @@ class SystemUptime(Plugin): - AgentPluginType = "sys" + AgentPluginType = "system" DEFAULT_CONFIG = {"uptime": str(60 * 5)} query_agent = "cat /proc/uptime | awk '{ print int($1) }'" diff --git a/mamonsu/tools/bootstrap/sql.py b/mamonsu/tools/bootstrap/sql.py index 3f9862b8..6983e5ad 100644 --- a/mamonsu/tools/bootstrap/sql.py +++ b/mamonsu/tools/bootstrap/sql.py @@ -133,7 +133,7 @@ $$ LANGUAGE SQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION mamonsu.archive_command_files() -RETURNS TABLE(COUNT_FILES BIGINT, SIZE_FILES BIGINT) AS $$ +RETURNS TABLE(files_count BIGINT, files_size BIGINT) AS $$ WITH values AS ( SELECT 4096/(ceil(pg_settings.setting::numeric/1024/1024)) AS segment_parts_count, @@ -146,13 +146,13 @@ ('x' || substring(pg_{10}_name(pg_current_{4}()) from 17 for 8))::bit(32)::int END AS current_wal_mod FROM pg_settings, pg_stat_archiver WHERE pg_settings.name = 'wal_segment_size') -SELECT greatest(coalesce((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1, 0), 0)::bigint AS count_files, -greatest(coalesce(((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1) * segment_size, 0), 0)::bigint AS size_files +SELECT greatest(coalesce((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1, 0), 0)::bigint AS files_count, +greatest(coalesce(((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1) * segment_size, 0), 0)::bigint AS files_size FROM values $$ LANGUAGE SQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION mamonsu.archive_stat() -RETURNS TABLE(ARCHIVED_COUNT BIGINT, FAILED_COUNT BIGINT) AS $$ +RETURNS TABLE(archived_count BIGINT, failed_count BIGINT) AS $$ SELECT archived_count, failed_count from pg_stat_archiver $$ LANGUAGE SQL SECURITY DEFINER; diff --git a/mamonsu/tools/zabbix_cli/dashboard.py b/mamonsu/tools/zabbix_cli/dashboard.py index 15183833..e66210f6 100644 --- a/mamonsu/tools/zabbix_cli/dashboard.py +++ b/mamonsu/tools/zabbix_cli/dashboard.py @@ -1,6 +1,6 @@ from datetime import datetime -from mamonsu.plugins.pgsql.archive_command import ArchiveCommand +from mamonsu.plugins.pgsql.archiver import Archiver from mamonsu.plugins.pgsql.bgwriter import BgWriter from mamonsu.plugins.pgsql.checkpoint import Checkpoint from mamonsu.plugins.pgsql.connections import Connections @@ -1052,7 +1052,7 @@ def generate_dashboard(template, uuid): PgLocks.graph_name, Instance.graphs_name["tuples"], PgBufferCache.graph_name, - ArchiveCommand.key.format("[" + ArchiveCommand.Items[2][0] + "]"), + Archiver.key.format("[" + Archiver.Items[2][0] + "]"), Databases.key_autovacumm.format("[]"), BgWriter.graph_name_buffers, BgWriter.graph_name_ws, @@ -1069,14 +1069,14 @@ def generate_dashboard(template, uuid): Memory.key.format("[" + Memory.Items[0][0] + "]"), Instance.key + Instance.Items[4][1], Instance.graphs_name["events"], - ArchiveCommand.key.format("[" + ArchiveCommand.Items[3][0] + "]"), - ArchiveCommand.key.format("[" + ArchiveCommand.Items[0][0] + "]"), + Archiver.key.format("[" + Archiver.Items[3][0] + "]"), + Archiver.key.format("[" + Archiver.Items[0][0] + "]"), Wal.key_non_active_slots.format("[]"), Oldest.key.format("[transaction_time]"), PgHealth.key_ping.format("[]"), PgHealth.key_version.format("[]"), Wal.key_replication.format("[sec]"), - ArchiveCommand.key.format("[" + ArchiveCommand.Items[1][0] + "]"), + Archiver.key.format("[" + Archiver.Items[1][0] + "]"), Instance.key + Instance.Items[6][1], Instance.key + Instance.Items[7][1], Connections.key.format("[total]"), diff --git a/mamonsu_win32.spec b/mamonsu_win32.spec index 17cbe02a..d7d5f6d0 100644 --- a/mamonsu_win32.spec +++ b/mamonsu_win32.spec @@ -8,7 +8,7 @@ a = Analysis(['mamonsu_win32.py'], binaries=[], datas=[], hiddenimports=[ - 'mamonsu.plugins.pgsql.archive_command', + 'mamonsu.plugins.pgsql.archiver', 'mamonsu.plugins.pgsql.bgwriter', 'mamonsu.plugins.pgsql.cfs', 'mamonsu.plugins.pgsql.checkpoint', diff --git a/service_win32.spec b/service_win32.spec index 4e1de348..318d4858 100644 --- a/service_win32.spec +++ b/service_win32.spec @@ -8,7 +8,7 @@ a = Analysis(['service_win32.py'], binaries=[], datas=[], hiddenimports=[ - 'mamonsu.plugins.pgsql.archive_command', + 'mamonsu.plugins.pgsql.archiver', 'mamonsu.plugins.pgsql.bgwriter', 'mamonsu.plugins.pgsql.cfs', 'mamonsu.plugins.pgsql.checkpoint', pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy