LAB: skaliertes Nextcloud-System
von Carsten Rieger|Aktualisiert 25. April 2021
Ziel dieses LAB ist es, die Ausfallsicherheit des Nextcloud Systems zu erhöhen, also eine skalierte und somit resistentere Nextcloud Umgebung zu betreiben, die sowohl ein Loadbalancing nutzt, als auch redundante Backendsysteme (MariaDB Master/Slave und Redis-Server Master/Slave) und ein zentrales NFS berücksichtigt.
Aufbau der Umgebung, basierend auf Ubunu 18.04.x LTS
nginx Loadbalancer: Externe/Öffentlich IP und 192.168.2.10
Nextcloud Backend 1: 192.168.2.11
Nextcloud Backend 2: 192.168.2.12
Master MariaDB und Redis-Slave: 192.168.2.20
Redis-Master und Slave MariaDB: 192.168.2.21
NFS Server für Daten: 192.168.2.30
Hinweis: Ich empfehle dringend den SSH Standardport 22 zu ändern – berücksichtigen Sie das bitte in den exemplarisch dargestellten Firewallsettings. Zudem empfehle ich auch den Einsatz von fail2ban und weitere Härtungsmaßnahmen, die nicht Gegenstand dieses LABs sind.
Installationsreihenfolge:
MariaDB Master/Slave
Redis Master/Slave
NFS
Nextcloud-Server
nginx Loadbalancer
1. MariaDB Master/Slave
Server: MariaDB Master- und Redis-Master
sudo -s
apt-get install software-properties-common -y
apt-key adv –recv-keys –keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
add-apt-repository ‘deb [arch=amd64] http://mirror.lstn.net/mariadb/repo/10.5/ubuntu bionic main’
apt update && apt install mariadb-server -y
ufw allow 3306/tcp
mysql_secure_installation
Server: MariaDB-Master und -Slave
systemctl enable mariadb
service mariadb stop
cd /etc/mysql/
cp my.cnf my.cnf.bak
vi my.cnf
[client]
default-character-set = utf8mb4
port = 3306
socket = /var/run/mysqld/mysqld.sock
[mysqld_safe]
log_error=/var/log/mysql/mysql_error.log
nice = 0
socket = /var/run/mysqld/mysqld.sock
[mysqld]
server-id = 01
replicate-do-db = nextcloud
basedir = /usr
#bind-address = 127.0.0.1
bind-address = 192.168.2.20
binlog_format = mixed
# binlog_format = ROW
bulk_insert_buffer_size = 16M
character-set-server = utf8mb4
collation-server = utf8mb4_general_ci
concurrent_insert = 2
connect_timeout = 5
datadir = /var/lib/mysql
default_storage_engine = InnoDB
expire_logs_days = 2
general_log_file = /var/log/mysql/mysql.log
general_log = 0
innodb_buffer_pool_size = 1024M
innodb_buffer_pool_instances = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 32M
innodb_max_dirty_pages_pct = 90
innodb_file_per_table = 1
innodb_open_files = 400
innodb_io_capacity = 4000
innodb_flush_method = O_DIRECT
key_buffer_size = 128M
lc_messages_dir = /usr/share/mysql
lc_messages = en_US
log_bin = /var/log/mysql/master-bin
# log_bin = /var/log/mysql/mariadb-bin
log_bin_index = /var/log/mysql/master-bin.index
# log_bin_index = /var/log/mysql/mariadb-bin.index
log_error = /var/log/mysql/mysql_error.log
log_slow_verbosity = query_plan
log_warnings = 2
long_query_time = 1
max_allowed_packet = 16M
max_binlog_size = 100M
max_connections = 300
max_heap_table_size = 64M
myisam_recover_options = BACKUP
myisam_sort_buffer_size = 512M
port = 3306
pid-file = /var/run/mysqld/mysqld.pid
query_cache_limit = 2M
query_cache_size = 64M
query_cache_type = 1
query_cache_min_res_unit = 2k
read_buffer_size = 2M
read_rnd_buffer_size = 1M
relay_log = /var/log/mysql/master-relay-bin
relay_log_index = /var/log/mysql/master-relay-bin.index
skip-external-locking
skip-name-resolve
slow_query_log_file = /var/log/mysql/mariadb-slow.log
slow-query-log = 1
socket = /var/run/mysqld/mysqld.sock
sort_buffer_size = 4M
table_open_cache = 400
thread_cache_size = 128
tmp_table_size = 64M
tmpdir = /tmp
transaction_isolation = READ-COMMITTED
# unix_socket=OFF
user = mysql
wait_timeout = 600
[mysqldump]
max_allowed_packet = 16M
quick
quote-names
[isamchk]
key_buffer = 16M
service mariadb restart
mysql -e “CREATE USER ‘nextcloud’@’localhost’ identified by ‘nextcloud’; flush privileges;”
mysql -e “CREATE USER ‘nextcloud’@’192.168.2.21’ identified by ‘nextcloud’; flush privileges;”
mysql -e “CREATE USER ‘nextcloud’@’192.168.2.20’ identified by ‘nextcloud’; flush privileges;”
mysql -e “grant replication slave on *.* to ‘nextcloud’@’192.168.2.21′;”
mysql -e “grant ALL PRIVILEGES on nextcloud.* to nextcloud@’192.168.2.20′; FLUSH privileges;”
mysql -e “grant ALL PRIVILEGES on nextcloud.* to nextcloud@’192.168.2.21′; FLUSH privileges;”
mysql -e “grant ALL PRIVILEGES on nextcloud.* to nextcloud@’localhost’; FLUSH privileges;”
mysql -e “SHOW MASTER STATUS\G;”
Server: MariaDB-Slave
systemctl enable mariadb
service mariad stop
cd /etc/mysql/
cp my.cnf my.cnf.bak
vi my.cnf
[client]
default-character-set = utf8mb4
port = 3306
socket = /var/run/mysqld/mysqld.sock
[mysqld_safe]
log_error=/var/log/mysql/mysql_error.log
nice = 0
socket = /var/run/mysqld/mysqld.sock
[mysqld]
server-id = 02
replicate-do-db = nextcloud
basedir = /usr
#bind-address = 127.0.0.1
bind-address = 192.168.2.21
binlog_format = mixed
# binlog_format = ROW
bulk_insert_buffer_size = 16M
character-set-server = utf8mb4
collation-server = utf8mb4_general_ci
concurrent_insert = 2
connect_timeout = 5
datadir = /var/lib/mysql
default_storage_engine = InnoDB
expire_logs_days = 2
general_log_file = /var/log/mysql/mysql.log
general_log = 0
innodb_buffer_pool_size = 1024M
innodb_buffer_pool_instances = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 32M
innodb_max_dirty_pages_pct = 90
innodb_file_per_table = 1
innodb_open_files = 400
innodb_io_capacity = 4000
innodb_flush_method = O_DIRECT
key_buffer_size = 128M
lc_messages_dir = /usr/share/mysql
lc_messages = en_US
log_bin = /var/log/mysql/slave-bin
# log_bin = /var/log/mysql/mariadb-bin
log_bin_index = /var/log/mysql/slave-bin.index
# log_bin_index = /var/log/mysql/mariadb-bin.index
log_error = /var/log/mysql/mysql_error.log
log_slow_verbosity = query_plan
log_warnings = 2
long_query_time = 1
max_allowed_packet = 16M
max_binlog_size = 100M
max_connections = 300
max_heap_table_size = 64M
myisam_recover_options = BACKUP
myisam_sort_buffer_size = 512M
port = 3306
pid-file = /var/run/mysqld/mysqld.pid
query_cache_limit = 2M
query_cache_size = 64M
query_cache_type = 1
query_cache_min_res_unit = 2k
read_buffer_size = 2M
read_rnd_buffer_size = 1M
relay_log = /var/log/mysql/slave-relay-bin
relay_log_index = /var/log/mysql/slave-relay-bin.index
skip-external-locking
skip-name-resolve
slow_query_log_file = /var/log/mysql/mariadb-slow.log
slow-query-log = 1
socket = /var/run/mysqld/mysqld.sock
sort_buffer_size = 4M
table_open_cache = 400
thread_cache_size = 128
tmp_table_size = 64M
tmpdir = /tmp
transaction_isolation = READ-COMMITTED
# unix_socket=OFF
user = mysql
wait_timeout = 600
[mysqldump]
max_allowed_packet = 16M
quick
quote-names
[isamchk]
key_buffer = 16M
service mariadb restart
Server: MariaDB Master
mysql -uroot -p
CREATE DATABASE nextcloud CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
flush tables with read lock;
show master status;
WICHTIG: diese mysql-Verbindung offen und bestehen lassen und eine weitere SSH Session starten:
mysqldump –single-transaction –routines -uroot -p*SECRET* nextcloud > nextcloud-dump.sql
mysql -e “SHOW MASTER STATUS\G;”
root@DBMASTER:~# mysql -e “SHOW MASTER STATUS\G;”
*************************** 1. row ***************************
File: master-bin.000001
Position: 8233
Binlog_Do_DB:
Binlog_Ignore_DB:
root@DBMASTER:~#
Server: MariaDB Slave
mysql -e “CREATE DATABASE nextcloud CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;”
mysql -uroot -p*SECRET* nextcloud < nextcloud-dump.sql service mariadb restart mysql -e “change master ‘master01′ to master_host=’192.168.2.20′, master_user=’nextcloud’, master_password=’nextcloud’, master_port=3306, master_log_file=’master-bin.000001, master_log_pos=8233, master_connect_retry=10, master_use_gtid=slave_pos;” mysql -e “start slave ‘master01’;” mysql -e “show slave ‘master01’ status\G;” Prüfen der Werte „Slave_IO_Running: Yes undSlave_SQL_Running: Yes„ root@DBSLAVE:~# mysql -e “show slave ‘master01’ status\G;” *************************** 1. row *************************** Slave_IO_State: Waiting for master to send event Master_Host: 192.168.2.11 Master_User: nextcloud Master_Port: 3306 Connect_Retry: 10 Master_Log_File: master-bin.000001 Read_Master_Log_Pos: 8233 Relay_Log_File: slave-relay-bin-master01.000002 Relay_Log_Pos: 1160 Relay_Master_Log_File: master-bin.000001 Slave_IO_Running: Yes Slave_SQL_Running: Yes Replicate_Do_DB: nextcloud Replicate_Ignore_DB: […] Prüfen des „synchronen“ Repliaktionsstatus: Server: MariaDB Master show variables like ‘gtid_binlog_pos’; +—————–+——–+ | Variable_name | Value | +—————–+——–+ | gtid_binlog_pos | 0-1-17 | +—————–+——–+ Server: MariaDB Slave show variables like ‘gtid_slave_pos’; +—————-+——–+ | Variable_name | Value | +—————-+——–+ | gtid_slave_pos | 0-1-17 | +—————-+——–+ Die Werte vom Master sollten identisch mit denen des Slaves sein. Server: MariaDB-Master und -Slave root@DBMASTER:~# ufw status verbose Status: active Logging: on (medium) Default: deny (incoming), allow (outgoing), disabled (routed) New profiles: skip To Action From — —— —- 22/tcp ALLOW IN Anywhere 3306 ALLOW IN 192.168.2.11 3306 ALLOW IN 192.168.2.12 22/tcp (v6) ALLOW IN Anywhere (v6) 2. Redis Master/Slave Server: Redis-Master und -Slave sudo -s apt install redis-server -y redis-cli ping PONG Server: Redis-Master vi /etc/redis/redis.conf bind 127.0.0.1 192.168.2.21 requirepass nextcloud systemctl restart redis-server.service systemctl enable redis-server.service Server: Redis-Slave vi /etc/redis/redis.conf bind 127.0.0.1 192.168.2.20 slaveof 192.168.2.21 6379 requirepass nextcloud masterauth nextcloud systemctl restart redis-server.service systemctl enable redis-server.service Server: Redis-Master root@DBSLAVE:~# redis-cli 127.0.0.1:6379> auth nextcloud
OK
127.0.0.1:6379> set MasterSlave-TEST ‘Carsten Rieger IT-Services – Quelle ist der Master’
OK
127.0.0.1:6379> quit
root@DBSLAVE:~#
Server: Redis-Slave
root@DBMASTER:~# redis-cli
127.0.0.1:6379> auth nextcloud
OK
127.0.0.1:6379> get MasterSlave-TEST
“Carsten Rieger IT-Services – Quelle ist der Master”
127.0.0.1:6379> quit
root@DBMASTER:~#
Server: Redis-Master und -Slave
root@DBMASTER:~# ufw status verbose
Status: active
Logging: on (medium)
Default: deny (incoming), allow (outgoing), disabled (routed)
New profiles: skip
To Action From
— —— —-
22/tcp ALLOW IN Anywhere
3306 ALLOW IN 192.168.2.11
3306 ALLOW IN 192.168.2.12
6379 ALLOW IN 192.168.2.11
6379 ALLOW IN 192.168.2.12
16379 ALLOW IN 192.168.2.11
16379 ALLOW IN 192.168.2.12
22/tcp (v6) ALLOW IN Anywhere (v6)
3. NFS Server
sudo -s
apt install nfs-kernel-server -y
mkdir -p /daten /nextcloud
chown -R www-data:www-data /daten /nextcloud
chmod -R 777 /daten /nextcloud
vi /etc/exports
/daten 192.168.2.11(rw,sync,anonuid=33,anongid=33,no_subtree_check)
/daten 192.168.2.12(rw,sync,anonuid=33,anongid=33,no_subtree_check)
/nextcloud 192.168.2.11(rw,sync,anonuid=33,anongid=33,no_subtree_check)
/nextcloud 192.168.2.12(rw,sync,anonuid=33,anongid=33,no_subtree_check)
exportfs -a
systemctl enable nfs-kernel-server
systemctl restart nfs-kernel-server
root@NFS:~# ufw status verbose
Status: active
Logging: on (medium)
Default: deny (incoming), allow (outgoing), disabled (routed)
New profiles: skip
To Action From
— —— —-
22/tcp ALLOW IN Anywhere
2049 ALLOW IN 192.168.2.11
2049 ALLOW IN 192.168.2.12
22/tcp (v6) ALLOW IN Anywhere (v6)
4. Nextcloud-Server
Server: Nextcloud Server 1 und 2
sudo -s
apt install nfs-common -y
mkdir /daten -p
chown -R www-data:www-data /daten
vi /etc/fstab
192.168.2.30:/daten /daten nfs auto,nofail,noatime,nolock,intr,tcp,actimeo=1800 0 0
192.168.2.30:/nextcloud /var/www/nextcloud nfs auto,nofail,noatime,nolock,intr,tcp,actimeo=1800 0 0
mount -a
Server: Nextcloud Server 1
vi /etc/nginx/conf.d/nextcloud-server1.conf
server {
server_name 192.168.2.11;
listen 192.168.2.11:80;
root /var/www/nextcloud/;
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
location ^~ / {
client_max_body_size 10G;
location / {
rewrite ^ /index.php;
}
location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ {
deny all;
}
location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) {
deny all;
}
location ~ ^\/(?:index|ipcheck|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) {
fastcgi_split_path_info ^(.+?\.php)(\/.*|)$;
set $path_info $fastcgi_path_info;
try_files $fastcgi_script_name =404;
include fastcgi_params;
include /etc/nginx/php_optimization.conf;
fastcgi_param REMOTE_ADDR $http_x_real_ip;
}
location ~ ^\/(?:updater|ocs-provider|ocm-provider)(?:$|\/) {
try_files $uri/ =404;
index index.php;
}
location ~ ^\/.+[^\/]\.(?:css|js|woff2?|svg|gif)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
location ~ ^\/.+[^\/]\.(?:png|html|ttf|ico|jpg|jpeg)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
}
}
Server: Nextcloud Server 2
vi /etc/nginx/conf.d/nextcloud-server2.conf
server {
server_name 192.168.2.12;
listen 192.168.2.12:80;
root /var/www/nextcloud/;
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
location ^~ / {
client_max_body_size 10G;
location / {
rewrite ^ /index.php;
}
location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ {
deny all;
}
location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) {
deny all;
}
location ~ ^\/(?:index|ipcheck|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) {
fastcgi_split_path_info ^(.+?\.php)(\/.*|)$;
set $path_info $fastcgi_path_info;
try_files $fastcgi_script_name =404;
include fastcgi_params;
include /etc/nginx/php_optimization.conf;
fastcgi_param REMOTE_ADDR $http_x_real_ip;
}
location ~ ^\/(?:updater|ocs-provider|ocm-provider)(?:$|\/) {
try_files $uri/ =404;
index index.php;
}
location ~ ^\/.+[^\/]\.(?:css|js|woff2?|svg|gif)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
location ~ ^\/.+[^\/]\.(?:png|html|ttf|ico|jpg|jpeg)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
}
}
Server: Nextcloud Server 1 und 2
sudo -u www-data vi /var/www/nextcloud/ipcheck.php
root@NEXTCLOUDSERVER#:~# curl https://ihre.domain.de/ipcheck.php
Test im Browser: curl https://ihre.domain.de/ipcheck.php
root@NEXTCLOUDSERVER#:~# ufw status verbose
Status: active
Logging: on (medium)
Default: deny (incoming), allow (outgoing), disabled (routed)
New profiles: skip
To Action From
— —— —-
22/tcp ALLOW IN Anywhere
80/tcp ALLOW IN 192.168.2.10
22/tcp (v6) ALLOW IN Anywhere (v6)
5. nginx Loadbalancer
sudo -s
vi /etc/nginx/conf.d/gateway.conf
upstream Nextcloud-Loadbalancer {
hash $remote_addr;
server 192.168.2.11; # <- Nextcloud Server 1
server 192.168.2.12; # <- Nextcloud Server 2
}
proxy_cache_path /tmp/nginx-cache levels=1:2 keys_zone=nextcloud_cache:10m max_size=10g inactive=60m use_temp_path=off;
server {
server_name ihre.domäne.de 192.168.2.10; # External/Public IP und Loadbalancer-IP
listen 80 default_server;
listen [::]:80 default_server;
location ^~ /.well-known/acme-challenge {
proxy_pass http://127.0.0.1:81;
proxy_set_header Host $host;
}
location / {
return 301 https://$server_name$request_uri;
}
}
server {
server_name ihre.domäne.de 192.168.2.10; # External/Public IP und Loadbalancer-IP
listen 443 ssl http2 default_server;
listen [::]:443 ssl http2 default_server;
root /var/www/nextcloud/;
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
include /etc/nginx/proxy.conf;
include /etc/nginx/header.conf;
client_max_body_size 10240M;
location ^~ / {
proxy_cache nextcloud_cache;
proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
proxy_cache_revalidate on;
proxy_cache_background_update on;
client_max_body_size 10G;
proxy_connect_timeout 3600;
proxy_send_timeout 3600;
proxy_read_timeout 3600;
send_timeout 3600;
proxy_buffering off;
proxy_request_buffering off;
proxy_max_temp_file_size 10240m;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://Nextcloud-Loadbalancer;
proxy_redirect off;
}
location = /.well-known/carddav {
return 301 $scheme://$host/remote.php/dav;
}
location = /.well-known/caldav {
return 301 $scheme://$host/remote.php/dav;
}
location = /.well-known/webfinger {
return 301 $scheme://$host/public.php?service=webfinger;
}
location ~ /(ocm-provider|ocs-provider)/ {
return 301 $scheme://$host/$1/;
}
}
root@LOADBALANCER:~# ufw status verbose
Status: active
Logging: on (medium)
Default: deny (incoming), allow (outgoing), disabled (routed)
New profiles: skip
To Action From
— —— —-
22/tcp ALLOW IN Anywhere
80/tcp ALLOW IN Anywhere
443/tcp ALLOW IN Anywhere
22/tcp (v6) ALLOW IN Anywhere (v6)
80/tcp (v6) ALLOW IN Anywhere (v6)
443/tcp (v6) ALLOW IN Anywhere (v6)