diff --git a/README.md b/README.md index db441db..c2aaa81 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +# Introduction + This repository contains an implementation of a monitoring dashboard of a Nextcloud system, including the application, the operating system, the database and the web server. The dashboard is to be hosted in a Zabbix system. + +## License + +GNU Affero General Public License (AGPL). See https://www.gnu.org/licenses/#AGPL. + diff --git a/params.json b/params.json new file mode 100644 index 0000000..3f912aa --- /dev/null +++ b/params.json @@ -0,0 +1,6 @@ +{ + "nc-token": "", + "nc_db": "nextcloud_db", + "nc_host": "localhost", + "nc_db_port": 3306 +} diff --git a/zabbix-agent-scripts/discover_apache2_user_agent_metrics.py b/zabbix-agent-scripts/discover_apache2_user_agent_metrics.py new file mode 100644 index 0000000..0cbdf83 --- /dev/null +++ b/zabbix-agent-scripts/discover_apache2_user_agent_metrics.py @@ -0,0 +1,41 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +import warnings + +import sys + +import json + +warnings.filterwarnings("ignore") + +norm_user_agents = json.load(open("/var/lib/zabbix/output/apache2_user_agent_metrics.json")) + +norm_user_agent_list = [] + +for norm_user_agent in norm_user_agents.keys(): + + norm_user_agent_list.append( + {"{#NORM_USER_AGENT}":norm_user_agent} + ) + + +print( + json.dumps(norm_user_agent_list) + ) diff --git a/zabbix-agent-scripts/discover_db_metrics.py b/zabbix-agent-scripts/discover_db_metrics.py index 726144e..57f91ee 100644 --- a/zabbix-agent-scripts/discover_db_metrics.py +++ b/zabbix-agent-scripts/discover_db_metrics.py @@ -1,3 +1,22 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + import pandas as pd import sys diff --git a/zabbix-agent-scripts/functions.py b/zabbix-agent-scripts/functions.py new file mode 100644 index 0000000..9e1c456 --- /dev/null +++ b/zabbix-agent-scripts/functions.py @@ -0,0 +1,88 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + + +import json + +def get_param(params_file,param_name): + + return json.load(open(params_file,"r"))[param_name] + + +def get_norm_user_agent(s): + ## EDGE + # Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.57 + ## Chrome + # Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 + ## Firefox + # Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:110.0) Gecko/20100101 Firefox/110.0 + ## Safari + # Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15 + + s = s.lower() + + ## Nextcloud Talk App + if "nextcloud-talk" in s: + if "android" in s: + return "nextcloud_talk_android" + elif "ios" in s: + return "nextcloud_talk_ios" + else: + return "nextcloud_talk_unknown" + + ## Nextcloud Files App + if "nextcloud-android" in s: + return "nextcloud_android" + + ## Browser + if "chrome" in s: + if "edg" in s: + return "edge_browser" + else: + return "chrome_browser" + if "firefox" in s: + return "firefox_browser" + if "macintosh" in s and "applewebkit" in s: + return "safari_browser" + + ## Desktop Client + if "mirall" in s: + if "linux" in s: + return "desktop_client_linux" + elif "macintosh" in s: + return "desktop_client_mac" + elif "windows" in s: + return "desktop_client_windows" + else: + return "desktop_client_unknown" + + ## DavX5, CardDAV / CalDAV sync agent on Android + if "davx5" in s: + return "davx5" + + ## Thunderbird, CardDAV / CalDAV sync agent on Desktop + if "thunderbird" in s: + return "thunderbird" + + ## Python scripting + if "python" in s: + return "python" + + return "other" + diff --git a/zabbix-agent-scripts/get_apache2_user_agent_metrics.py b/zabbix-agent-scripts/get_apache2_user_agent_metrics.py index 20cc2d7..eebe106 100644 --- a/zabbix-agent-scripts/get_apache2_user_agent_metrics.py +++ b/zabbix-agent-scripts/get_apache2_user_agent_metrics.py @@ -1,11 +1,30 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + import pandas as pd from datetime import datetime -import sys - import json +from functions import get_norm_user_agent + dt = datetime.now().strftime("%Y%m%d") df = pd.read_csv( @@ -17,70 +36,6 @@ df.fillna("not-found",inplace=True) -## EDGE -# Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.57 -## Chrome -# Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 -## Firefox -# Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:110.0) Gecko/20100101 Firefox/110.0 -## Safari -# Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15 - -def get_norm_user_agent(s): - - s = s.lower() - - ## Nextcloud Talk App - if "nextcloud-talk" in s: - if "android" in s: - return "nextcloud_talk_android" - elif "ios" in s: - return "nextcloud_talk_ios" - else: - return "nextcloud_talk_unknown" - - ## Nextcloud Files App - if "nextcloud-android" in s: - return "nextcloud_android" - - ## Browser - if "chrome" in s: - if "edg" in s: - return "edge_browser" - else: - return "chrome_browser" - if "firefox" in s: - return "firefox_browser" - if "macintosh" in s and "applewebkit" in s: - return "safari_browser" - - ## Desktop Client - if "mirall" in s: - if "linux" in s: - return "desktop_client_linux" - elif "macintosh" in s: - return "desktop_client_mac" - elif "windows" in s: - return "desktop_client_windows" - else: - return "desktop_client_unknown" - - ## DavX5, CardDAV / CalDAV sync agent on Android - if "davx5" in s: - return "davx5" - - ## Thunderbird, CardDAV / CalDAV sync agent on Desktop - if "thunderbird" in s: - return "thunderbird" - - ## Python scripting - if "python" in s: - return "python" - - return "other" - - - df["norm_user_agent"] = df.user_agent.apply(get_norm_user_agent) df_agg = df.value_counts("norm_user_agent") @@ -89,3 +44,4 @@ df_agg.to_dict(), open("/var/lib/zabbix/output/apache2_user_agent_metrics.json","w") ) + diff --git a/zabbix-agent-scripts/get_db_metrics.py b/zabbix-agent-scripts/get_db_metrics.py index f880658..f3d121c 100644 --- a/zabbix-agent-scripts/get_db_metrics.py +++ b/zabbix-agent-scripts/get_db_metrics.py @@ -1,9 +1,30 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + import pandas as pd import mariadb as mdb import configparser +from functions import get_param + config = configparser.ConfigParser() config.read("/var/lib/zabbix/.my.cnf") @@ -12,18 +33,18 @@ nc_usr_pwd = config["client"].get("password") -## The following variables must be filled in by the user +param_file = "/var/lib/zabbix/params.json" -nc_db = "nextcloud_db" +nc_db = get_param(param_file,"nc_db") -#tables_to_monitor = "'oc_filecache','oc_comments','oc_authtoken','oc_circles_event','oc_share'" +nc_host = get_param(param_file,"nc_host") -## +nc_db_port = get_param(param_file,"nc_db_port") conn = mdb.connect( user=nc_usr, password=nc_usr_pwd, - host="localhost", - port=3306, + host=nc_host, + port=nc_db_port, database=nc_db ) @@ -46,4 +67,4 @@ ) df.to_json( "/var/lib/zabbix/output/db_metrics.json", - orient='columns') + orient='columns') diff --git a/zabbix-agent-scripts/get_nc_metrics.py b/zabbix-agent-scripts/get_nc_metrics.py index 28c3765..055b270 100644 --- a/zabbix-agent-scripts/get_nc_metrics.py +++ b/zabbix-agent-scripts/get_nc_metrics.py @@ -1,12 +1,33 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + import json import requests -nextcloud_token = "7iz11gm77ivdf24hnfulycpa" +from functions import get_param -metric_values = json.loads(requests.get("http://localhost/ocs/v2.php/apps/serverinfo/api/v1/info?format=json", +nc_token = get_param("/var/lib/zabbix/params.json", "nc-token") + +metric_values = json.loads(requests.get("http://localhost/ocs/v2.php/apps/serverinfo/api/v1/info?format=json&skipApps=false", verify=False, - headers={"NC-Token": "%s" % nextcloud_token}).content + headers={"NC-Token": "%s" % nc_token}).content ) with open("/var/lib/zabbix/output/nc_metrics.json","w") as fl: diff --git a/zabbix-agent-scripts/get_nc_quota_metrics.py b/zabbix-agent-scripts/get_nc_quota_metrics.py index 4c198d1..9c8dce9 100644 --- a/zabbix-agent-scripts/get_nc_quota_metrics.py +++ b/zabbix-agent-scripts/get_nc_quota_metrics.py @@ -1,3 +1,22 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + import subprocess import pandas as pd diff --git a/zabbix-agent-scripts/get_storage_metrics.py b/zabbix-agent-scripts/get_storage_metrics.py index fe01da0..6f645b0 100644 --- a/zabbix-agent-scripts/get_storage_metrics.py +++ b/zabbix-agent-scripts/get_storage_metrics.py @@ -1,25 +1,68 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + import subprocess import json -datadir = subprocess.check_output(["occ", "config:system:get", "datadirectory"]).strip(b"\n").decode() +import io -total_size = subprocess.check_output(["du","-b", "-d0", datadir]).split(b"\t")[0].decode() +import pandas as pd -total_size_gb = int(total_size)/1024./1024./1024. -total_size_dict = { "total_size_gb": total_size_gb } +datadir = subprocess.check_output( + ["occ", "config:system:get", "datadirectory"] + ).strip(b"\n").decode() +total_size = subprocess.check_output( + ["du","-b", "-d0", datadir] + ).split(b"\t")[0].decode() + +gb_conversion_factor = 1./(1024.*1024.*1024.) + +datadirectory_total_size_gb = int(total_size)*gb_conversion_factor + +df_command_output = subprocess.check_output( + ["df", "-B1", "--output=size,avail,pcent", datadir] + ).strip(b"\n").decode() + +df = pd.read_csv( + io.StringIO(df_command_output), + sep="\s+", + names=["total","avail","pused"], + skiprows=1 + ) + +partition_total_capacity_gb = (df.total.values[0])*gb_conversion_factor + +partition_available_capacity_gb = (df.avail.values[0])*gb_conversion_factor + +partition_used_capacity_percent = int(df.pused.values[0].strip('%')) + +storage_metrics = { + "datadirectory_total_size_gb": datadirectory_total_size_gb, + "partition_total_capacity_gb": partition_total_capacity_gb, + "partition_available_capacity_gb": partition_available_capacity_gb, + "partition_used_capacity_percent": partition_used_capacity_percent + } with open("/var/lib/zabbix/output/storage_metrics.json","w") as fl: - json.dump(total_size_dict,fl) - - - - - - - - + json.dump(storage_metrics,fl) diff --git a/zabbix-agent-scripts/read_apache2_user_agent_metrics.py b/zabbix-agent-scripts/read_apache2_user_agent_metrics.py index c276baf..63ca831 100644 --- a/zabbix-agent-scripts/read_apache2_user_agent_metrics.py +++ b/zabbix-agent-scripts/read_apache2_user_agent_metrics.py @@ -1,3 +1,22 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + import json import sys diff --git a/zabbix-agent-scripts/read_db_metrics.py b/zabbix-agent-scripts/read_db_metrics.py index d749c5f..951f379 100644 --- a/zabbix-agent-scripts/read_db_metrics.py +++ b/zabbix-agent-scripts/read_db_metrics.py @@ -1,3 +1,22 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + import pandas as pd import sys diff --git a/zabbix-agent-scripts/read_nc_metrics.py b/zabbix-agent-scripts/read_nc_metrics.py index 1f0a266..eabbdc6 100644 --- a/zabbix-agent-scripts/read_nc_metrics.py +++ b/zabbix-agent-scripts/read_nc_metrics.py @@ -1,3 +1,22 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + ## warnings are disabled as they confuse Zabbix import warnings diff --git a/zabbix-agent-scripts/read_storage_metrics.py b/zabbix-agent-scripts/read_storage_metrics.py index 0fd0ed5..ec2ea39 100644 --- a/zabbix-agent-scripts/read_storage_metrics.py +++ b/zabbix-agent-scripts/read_storage_metrics.py @@ -1,3 +1,22 @@ +# +# @copyright Copyright (c) 2024, Pietro Marini (pmarini@rcasys.com) +# +# @license GNU AGPL version 3 or any later version +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + ## warnings are disabled as they confuse Zabbix import warnings diff --git a/zabbix-templates/nextcloud-zabbix-template.json b/zabbix-templates/nextcloud-zabbix-template.json index e87270e..7ffcb10 100644 --- a/zabbix-templates/nextcloud-zabbix-template.json +++ b/zabbix-templates/nextcloud-zabbix-template.json @@ -14,7 +14,7 @@ "templates": [ { "uuid": "eddea30a51d94d56875d2dd78fab8120", - "template": "Nextcloud", + "template": "Nextcloud by RCA Systems", "name": "Nextcloud", "description": "Nextcloud by RCA Systems", "vendor": { diff --git a/zabbix_agentd.conf.d/zabbix-agent-nextcloud.conf b/zabbix_agentd.conf.d/zabbix-agent-nextcloud.conf index cdc77d1..e1616c1 100644 --- a/zabbix_agentd.conf.d/zabbix-agent-nextcloud.conf +++ b/zabbix_agentd.conf.d/zabbix-agent-nextcloud.conf @@ -4,4 +4,5 @@ UserParameter=db_full_version[*],mysql --version UserParameter=db_metric[*],/opt/pyvenv/ncmonitor/bin/python /var/lib/zabbix/scripts/read_db_metrics.py $1 $2 UserParameter=db_metric_discovery,/opt/pyvenv/ncmonitor/bin/python /var/lib/zabbix/scripts/discover_db_metrics.py +UserParameter=apache2_user_agent_metric_discovery,/opt/pyvenv/ncmonitor/bin/python /var/lib/zabbix/scripts/discover_apache2_user_agent_metrics.py UserParameter=apache2_user_agent_metric[*],/opt/pyvenv/ncmonitor/bin/python /var/lib/zabbix/scripts/read_apache2_user_agent_metrics.py $1