【实战】使用Nginx实现UDP&TCP协议 Syslog 负载均衡

Nginx

使用Nginx实现UDP/TCP协议Syslog 反向代理(负载均衡)

1 说明

Nginx 从1.9.13起开始发布 ngx_stream_core_module 模块不仅能支持 TCP 代理及负载均衡同时还支持 UDP 协议。

2 安装和配置Nginx

2.1 使用yum安装Nginx依耐包

1
[root@other-server opt]$ yum -y install proc* openssl* pcre*

2.2 使用wget下载Nginx源代码安装包

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@other-server opt]$ wget https://nginx.org/download/nginx-1.25.2.tar.gz
--2023-08-30 16:34:13--  https://nginx.org/download/nginx-1.25.2.tar.gz
Resolving nginx.org (nginx.org)... 52.58.199.22, 3.125.197.172, 2a05:d014:edb:5702::6, ...
Connecting to nginx.org (nginx.org)|52.58.199.22|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1214903 (1.2M) [application/octet-stream]
Saving to: 'nginx-1.25.2.tar.gz'

100%[===================================================================================================>] 1,214,903   1.17MB/s   in 1.0s 

2023-08-30 16:34:15 (1.17 MB/s) - 'nginx-1.25.2.tar.gz' saved [1214903/1214903]
[root@other-server opt]$ ll
-rw-r--r--  1 root root 1214903 Aug 16 01:36 nginx-1.25.2.tar.gz

2.3 解压Nginx源代码安装包

1
2
3
4
[root@other-server opt]$ tar zxvf nginx-1.25.2.tar.gz
[root@other-server opt]$ ll
drwxr-xr-x  8 rsyslog rsyslog     158 Aug 16 01:03 nginx-1.25.2
-rw-r--r--  1 root    root    1214903 Aug 16 01:36 nginx-1.25.2.tar.gz

2.4 创建nginx用户和用户组

1
2
3
4
5
6
7
8
[root@other-server ]$ groupadd nginx  # 创建nginx用户组
[root@other-server ]$ useradd -m -g nginx nginx  # 创建用户,并指定用户组为nginx
[root@other-server ]$ passwd nginx  # 给nginx用户设置密码
Changing password for user nginx.
New password: 
BAD PASSWORD: The password is shorter than 8 characters
Retype new password: 
passwd: all authentication tokens updated successfully.

2.5 开始编译安装Nginx

详细的Nginx编译安装命令：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
./configure \
--prefix=/usr/local/nginx \
--sbin-path=/usr/local/nginx \
--conf-path=/usr/local/nginx/conf/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/log/nginx/nginx.pid \
--lock-path=/var/log/nginx/nginx.lock \
--http-client-body-temp-path=/var/log/nginx/client_temp \
--http-proxy-temp-path=/var/log/nginx/proxy_temp \
--http-fastcgi-temp-path=/var/log/nginx/fastcgi_temp \
--http-uwsgi-temp-path=/var/log/nginx/uwsgi_temp \
--http-scgi-temp-path=/var/log/nginx/scgi_temp \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_sub_module \
--with-http_dav_module \
--with-http_flv_module \
--with-http_mp4_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_random_index_module \
--with-http_secure_link_module \
--with-http_stub_status_module \
--with-http_auth_request_module \
--with-threads \
--with-stream \
--with-stream_ssl_module \
--with-mail \
--with-mail_ssl_module \
--with-file-aio \
--with-http_v2_module \
--with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic'

2.5.1 进入/opt/nginx-1.25.2目录

1
[root@other-server opt]$ cd /opt/nginx-1.25.2

2.5.2 开始编译安装nginx

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
[root@other-server nginx-1.25.2]$ ./configure \
--prefix=/usr/local/nginx \
--sbin-path=/usr/local/nginx/sbin \
--conf-path=/usr/local/nginx/conf/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/log/nginx/nginx.pid \
--lock-path=/var/log/nginx/nginx.lock \
--http-client-body-temp-path=/var/log/nginx/client_temp \
--http-proxy-temp-path=/var/log/nginx/proxy_temp \
--http-fastcgi-temp-path=/var/log/nginx/fastcgi_temp \
--http-uwsgi-temp-path=/var/log/nginx/uwsgi_temp \
--http-scgi-temp-path=/var/log/nginx/scgi_temp \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_sub_module \
--with-http_dav_module \
--with-http_flv_module \
--with-http_mp4_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_random_index_module \
--with-http_secure_link_module \
--with-http_stub_status_module \
--with-http_auth_request_module \
--with-threads \
--with-stream \
--with-stream_ssl_module \
--with-mail \
--with-mail_ssl_module \
--with-file-aio \
--with-http_v2_module \
--with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic'
checking for OS
    + Linux 3.10.0-1160.76.1.el7.x86_64 x86_64
checking for C compiler ... found
    + using GNU C compiler
    + gcc version: 4.8.5 20150623 (Red Hat 4.8.5-44) (GCC) 
checking for gcc -pipe switch ... found
checking for -Wl,-E switch ... found
······省略······
Configuration summary
    + using threads
    + using system PCRE2 library
    + using system OpenSSL library
    + using system zlib library

    nginx path prefix: "/usr/local/nginx"
    nginx binary file: "/usr/local/nginx"
    nginx modules path: "/usr/local/nginx/modules"
    nginx configuration prefix: "/usr/local/nginx/conf"
    nginx configuration file: "/usr/local/nginx/conf/nginx.conf"
    nginx pid file: "/var/log/nginx/nginx.pid"
    nginx error log file: "/var/log/nginx/error.log"
    nginx http access log file: "/var/log/nginx/access.log"
    nginx http client request body temporary files: "/var/log/nginx/client_temp"
    nginx http proxy temporary files: "/var/log/nginx/proxy_temp"
    nginx http fastcgi temporary files: "/var/log/nginx/fastcgi_temp"
    nginx http uwsgi temporary files: "/var/log/nginx/uwsgi_temp"
    nginx http scgi temporary files: "/var/log/nginx/scgi_temp"
# 检查通过！！！

2.5.3 开始make && make install

1
2
3
4
5
6
7
8
9
10
11
12
[root@other-server nginx-1.25.2]$ make && make install
make -f objs/Makefile
make[1]: Entering directory '/opt/nginx-1.25.2'
cc -c -pipe  -O -W -Wall -Wpointer-arith -Wno-unused-parameter -Werror -g -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic -I src/core -I src/event -I src/event/modules -I src/event/quic -I src/os/unix -I objs \
    -o objs/src/core/nginx.o \
    src/core/nginx.c
······省略······
test -d '/usr/local/nginx/html' \
    || cp -R html '/usr/local/nginx'
test -d '/var/log/nginx' \
    || mkdir -p '/var/log/nginx'
make[1]: Leaving directory '/opt/nginx-1.25.2'

2.5.4 安装完成 - 查看nginx版本信息

1
2
3
[root@other-server nginx-1.25.2]$ /usr/local/nginx/nginx -v
nginx version: nginx/1.25.2
[root@other-server nginx-1.25.2]$

2.6 设置Nginx环境变量

2.6.1 修改/etc/profile配置文件添加$NGINX_HOME环境变量

修改/etc/profile配置文件，在文件最末尾处添加以下内容完成$NGINX_HOME环境变量设置

export NGINX_HOME=”/usr/local/nginx”

export PATH=$PATH:$NGINX_HOME/sbin 

1
2
3
4
5
[root@other-server ]$ cat /etc/profile
export NGINX_HOME="/usr/local/nginx"
export PATH=$PATH:$NGINX_HOME/sbin
# 刷新/etc/profile配置文件
[root@other-server ]$ source /etc/profile

2.6.2 测试nginx环境变量是否生效，查看nginx版本信息

1
2
[root@other-server opt]$ nginx -v
nginx version: nginx/1.25.2

2.6.3 nginx相关目录映射

2.6.3.1 创建软链接
1
[root@other-server opt]$ ln -s /usr/local/nginx/conf /etc/nginx/conf
2.6.3.2 查看软链接创建状态

补充：删除软链接：使用unlink命令；如unlink nginx

1
2
3
[root@other-server nginx]$ ll /etc/nginx
total 0
lrwxrwxrwx 1 root root 22 Aug 30 22:01 conf - /usr/local/nginx/conf/

2.7 创建nginx.service使用systemctl管理

创建/usr/lib/systemd/system/nginx.service配置文件，使用systemctl管理Nginx服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[root@OtherServer opt]$ cat /usr/lib/systemd/system/nginx.service
[Unit]
Description=The nginx HTTP and reverse proxy server
After=network-online.target remote-fs.target nss-lookup.target
Wants=network-online.target

[Service]
Type=forking
PIDFile=/var/log/nginx/nginx.pid

# Nginx will fail to start if /run/nginx.pid already exists but has the wrong
# SELinux context. This might happen when running `nginx -t` from the cmdline.
# https://bugzilla.redhat.com/show_bug.cgi?id=1268621

ExecStartPre=/usr/local/nginx/sbin/nginx -t -c /usr/local/nginx/conf/nginx.conf
ExecStart=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
ExecReload=/usr/local/nginx/sbin/nginx -s reload
ExecStop=/usr/local/nginx/sbin/nginx -s stop
ExecQuit=/usr/local/nginx/sbin/nginx -s quit
PrivateTmp = true

[Install]
WantedBy=multi-user.target

3 创建Nginx UDP/TCP协议转发配置文件

3.1 nginx.conf

stream nginx.conf 主配置文件,加载514.conf配置文件。

1
2
3
4
5
6
7
8
9
stream {  # 需要与http同一级
	log_format basic 'time=[$time_local] remote_addr=$remote_addr '  # 设置basic日志格式
					'protocol=$protocol '
					'status=$status '
					'bytes_sent=$bytes_sent '
					'bytes_received=$bytes_received '
					'session_time=$session_time'
	include /usr/local/nginx/conf/stream/*.conf;  # 加载udp.d目录下的配置文件
}

3.2 514.conf

514 端口转发配置文件[负载均衡]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
upstream udp_514_server {
    # 希负载均衡策略可以通过客户端 IP（$remote_addr）实现简单的会话保持，其可将同一IP客户端始终转发给同一台后端服务器。
    hash $remote_addr;    
    server 10.10.0.102:514 weight=5;
    server 10.10.0.103:514 weight=5;
    # weight 权重值
    # udp不建议添加<max_fails>和<fail_timeout>参数
}

upstream tcp_514_server {
    # 希负载均衡策略可以通过客户端 IP（$remote_addr）实现简单的会话保持，其可将同一IP客户端始终转发给同一台后端服务器。
    hash $remote_addr;
    server 10.10.0.102:514 weight=5 max_fails=3 fail_timeout=30s;
    server 10.10.0.103:514 weight=5 max_fails=3 fail_timeout=30s;
    # weight 权重值
    # max_fails：在服务器被标记为不可用的时间内必须发生的失败尝试次数（默认为1次尝试）
    # fail_timeout：多次尝试失败而将服务器标记为不可用的时间，以及将服务器标记为不可用的时间（默认为10秒）
    # udp不建议添加<max_fails>和<fail_timeout>参数
}

# udp 514 端口代理
server {
    listen 514 udp;                                 # 监听udp 514端口
    proxy_timeout           1s;                     # 获取被代理服务器的响应最大超时时间为1s
    proxy_connect_timeout   1s;                     # 与被代理服务器建立连接的超时时间为1s
    access_log              /var/log/nginx/udp_514_server.log  basic;
    error_log               /var/log/nginx/udp_error_514_server.log;
    proxy_pass              udp_514_server;         # 代理upstream名称
}

# tcp 514 端口代理
server {
    listen 514;                                         # 监听tcp 514端口
    proxy_timeout               1s;                     # 获取被代理服务器的响应最大超时时间为1s
    proxy_connect_timeout       1s;                     # 与被代理服务器建立连接的超时时间为1s
    proxy_next_upstream         on;                     # 当被代理的服务器返回错误或超时时，将未返回响应的客户端连接请求传递给upstream中的下一个服务器
    proxy_next_upstream_tries   3;                      # 转发尝试请求最多3次
    proxy_next_upstream_timeout 10s;                    # 总尝试超时时间为10s
    proxy_socket_keepalive      on;                     # 开启SO_KEEPALIVE选项进行心跳检测
    access_log                  /var/log/nginx/tcp_514_server.log  basic;
    error_log                   /var/log/nginx/tcp_error_514_server.log;
    proxy_pass                  tcp_514_server;         # 代理upstream名称
}

3.3 重启Nginx服务使配置生效

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@other-server conf]$ systemctl status nginx
* nginx.service - The nginx HTTP and reverse proxy server
   Loaded: loaded (/etc/systemd/system/nginx.service; enabled; vendor preset: disabled)
   Active: active (running) since Wed 2023-08-30 22:46:07 CST; 7s ago
  Process: 52593 ExecStop=/usr/local/nginx/sbin/nginx -s stop (code=exited, status=1/FAILURE)
  Process: 106075 ExecStart=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf (code=exited, status=0/SUCCESS)
  Process: 106073 ExecStartPre=/usr/local/nginx/sbin/nginx -t -c /usr/local/nginx/conf/nginx.conf (code=exited, status=0/SUCCESS)
 Main PID: 106078 (nginx)
    Tasks: 5
   Memory: 3.4M
   CGroup: /system.slice/nginx.service
           |-106078 nginx: master process /usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
           |-106079 nginx: worker process
           |-106080 nginx: worker process
           |-106081 nginx: worker process
           `-106082 nginx: worker process

Aug 30 22:46:07 other-server systemd[1]: Starting The nginx HTTP and reverse proxy server...
Aug 30 22:46:07 other-server nginx[106073]: nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
Aug 30 22:46:07 other-server nginx[106073]: nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
Aug 30 22:46:07 other-server systemd[1]: Started The nginx HTTP and reverse proxy server.
[root@other-server conf]$ systemctl enable nginx #  设置开机启动

4 实现效果

1693497714425

Nginx > Linux > syslog > syslog-ng > 实战
#Linux #Nginx #端口转发 #TCP #UDP #Syslog #负载均衡
【实战】使用Nginx实现UDP&TCP协议 Syslog 负载均衡
https://hesc.info/d125840a56b4/
作者
需要哈气的纸飞机
发布于
2023年9月1日
许可协议