Fastdfs生产环境完整部署文档2016-01-21
系统环境：CentOS release 6.7 (Final)
IP地址和主机名对应关系
#--------------------------------------------------------------
192.168.1.79 f1
192.168.1.80 f2
192.168.1.81 f3
192.168.1.82 f4
192.168.1.83 f5
192.168.1.84 f6
192.168.1.85 f7
192.168.1.86 f8
#--------------------------------------------------------------
---------------------------------------------------------------------------------------------
角色分配：
-----------------------------------------------------------------
2 个 tracker server 192.168.1.79 192.168.1.80
2 个 group 每组内冗余两个副本
group 1
-----------------------------------------------------------------
192.168.1.81 f3
192.168.1.82 f4
-----------------------------------------------------------------
group2
-----------------------------------------------------------------
192.168.1.83 f5
192.168.1.84 f6
group3[模拟扩容使用]
-----------------------------------------------------------------
192.168.1.85 f7
192.168.1.86 f8
数据目录和日志元数据目录
----------------------------------------------------------------
/data/fastdfs_data
一：开始部署
1：关闭相关服务(所有机器)
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
chkconfig iptables off
mkdir -p /data/fastdfs_data
yum install -y wget
/bin/mv /etc/yum.repos.d/* /tmp/
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.163.com/.help/CentOS6-Base-163.repo
yum makecache
yum install -y gettext gettext-devel libXft libXft-devel libXpm libXpm-devel automake autoconf libXtst-devel gtk+-devel gcc gcc-c++ zlib-devel libpng-devel gtk2-devel glib-devel pcre* gcc git
2：安装libfastcommon and fastdfs(所有机器)
mkdir fastdfs
cd fastdfs
cd libfastcommon/
./make.sh
./make.sh install
cd ..
tar zvfx V5.05.tar.gz
cd fastdfs-5.05/
sed -i 's#/usr/local/bin/#/usr/bin/#g' init.d/fdfs_storaged
sed -i 's#/usr/local/bin/#/usr/bin/#g' init.d/fdfs_trackerd
./make.sh
./make.sh install
cd ..
3:配置f1 f2为tracker服务 两台机器部署相同
vim /etc/fdfs/tracker.conf
#---------------------------------------------------------------------------------------------------------------------------------
disabled=false
bind_addr=
port=22122
connect_timeout=30
bind_addr=
port=22122
connect_timeout=30
network_timeout=60
base_path=/data/fastdfs_data
max_connections=256
accept_threads=1
work_threads=4
store_lookup=2
store_group=group2
store_server=0
store_path=0
download_server=0
reserved_storage_space = 10%
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
sync_log_buff_interval = 10
check_active_interval = 120
thread_stack_size = 64KB
storage_ip_changed_auto_adjust = true
storage_sync_file_max_delay = 86400
storage_sync_file_max_time = 300
use_trunk_file = false
slot_min_size = 256
slot_max_size = 16MB
trunk_file_size = 64MB
trunk_create_file_advance = false
trunk_create_file_time_base = 02:00
trunk_create_file_interval = 86400
trunk_create_file_space_threshold = 20G
trunk_init_check_occupying = false
trunk_init_reload_from_binlog = false
trunk_compress_binlog_min_interval = 0
use_storage_id = false
storage_ids_filename = storage_ids.conf
id_type_in_filename = ip
store_slave_file_use_link = false
rotate_error_log = false
error_log_rotate_time=00:00
rotate_error_log_size = 0
log_file_keep_days = 0
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.server_port=80
http.check_alive_interval=30
http.check_alive_type=tcp
accept_threads=1
work_threads=4
store_lookup=2
store_group=group2
store_server=0
store_path=0
download_server=0
reserved_storage_space = 10%
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
sync_log_buff_interval = 10
check_active_interval = 120
thread_stack_size = 64KB
storage_ip_changed_auto_adjust = true
storage_sync_file_max_delay = 86400
storage_sync_file_max_time = 300
use_trunk_file = false
slot_min_size = 256
slot_max_size = 16MB
trunk_file_size = 64MB
trunk_create_file_advance = false
trunk_create_file_time_base = 02:00
trunk_create_file_interval = 86400
trunk_create_file_space_threshold = 20G
trunk_init_check_occupying = false
trunk_init_reload_from_binlog = false
trunk_compress_binlog_min_interval = 0
use_storage_id = false
storage_ids_filename = storage_ids.conf
id_type_in_filename = ip
store_slave_file_use_link = false
rotate_error_log = false
error_log_rotate_time=00:00
rotate_error_log_size = 0
log_file_keep_days = 0
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.server_port=80
http.check_alive_interval=30
http.check_alive_type=tcp
http.check_alive_uri=/status.html
#---------------------------------------------------------------------------------------------------------------------------------
vim /etc/fdfs/client.conf
#--------------------------------------------------------------------------------------------------------------------------------
connect_timeout=30
network_timeout=60
base_path=/data/fastdfs_data
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
log_level=info
use_connection_pool = false
connection_pool_max_idle_time = 3600
load_fdfs_parameters_from_tracker=false
use_storage_id = false
storage_ids_filename = storage_ids.conf
tracker_server=192.168.1.80:22122
log_level=info
use_connection_pool = false
connection_pool_max_idle_time = 3600
load_fdfs_parameters_from_tracker=false
use_storage_id = false
storage_ids_filename = storage_ids.conf
http.tracker_server_port=80
#--------------------------------------------------------------------------------------------------------------------------------
/etc/init.d/fdfs_trackerd start
echo "/etc/init.d/fdfs_trackerd start" >> /etc/rc.local
4:配置group1 (f3 f4 配置相同)
cp /etc/fdfs/storage.conf.sample /etc/fdfs/storage.conf
vim /etc/fdfs/storage.conf
#---------------------------------------------------------------------------------------------------------------------------------
disabled=false
group_name=group1
bind_addr=
client_bind=true
port=23000
connect_timeout=30
network_timeout=60
heart_beat_interval=30
stat_report_interval=60
base_path=/data/fastdfs_data
max_connections=256
buff_size = 256KB
accept_threads=1
work_threads=4
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/data/fastdfs_data
subdir_count_per_path=256
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = false
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
group_name=group1
bind_addr=
client_bind=true
port=23000
connect_timeout=30
network_timeout=60
heart_beat_interval=30
stat_report_interval=60
base_path=/data/fastdfs_data
max_connections=256
buff_size = 256KB
accept_threads=1
work_threads=4
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/data/fastdfs_data
subdir_count_per_path=256
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = false
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
http.server_port=80
#---------------------------------------------------------------------------------------------------------------------------------
#启动并加入开机启动项
/etc/init.d/fdfs_storaged restart
echo "/etc/init.d/fdfs_storaged restart" >> /etc/rc.local
storage server 安装 nginx
tar zvfx nginx-1.8.0.tar.gz
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx-1.8.0 \
--with-http_gzip_static_module \
--with-http_gunzip_module \
--add-module=../fastdfs-nginx-module/src
make
make install
cd ..
配置nginx
cp fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/
cp fastdfs-5.05/conf/{anti-steal.jpg,http.conf,mime.types} /etc/fdfs/
touch /data/logs/mod_fastdfs.log
vim nginx.conf
#-----------------------------------------------------------------------------------------------------------------------------------
user root root;
worker_processes 8;
worker_processes 8;
error_log /data/logs/error.log crit;
pid /usr/local/nginx-1.8.0/logs/nginx.pid;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
worker_rlimit_nofile 51200;
events {
use epoll;
worker_connections 51200;
}
http {
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
proxy_headers_hash_max_size 51200;
proxy_headers_hash_bucket_size 6400;
client_header_buffer_size 4k;
large_client_header_buffers 4 32k;
client_max_body_size 1M;
sendfile on;
tcp_nopush on;
keepalive_timeout 300;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 256k;
fastcgi_intercept_errors on;
server_tokens off;
client_body_buffer_size 256k;
send_timeout 3m;
client_header_timeout 3m;
client_body_timeout 3m;
proxy_ignore_client_abort on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
include vhosts/*.conf;
}
#-----------------------------------------------------------------------------------------------------------------------------------
mkdir -p /usr/local/nginx-1.8.0/conf/vhosts
vim /usr/local/nginx-1.8.0/conf/vhosts/fastdfs.conf
#----------------------------------------------------------------------------------------------------------------------------------
server {
listen 80;
server_name fastdfs.hichao.com;
location /group1/M00 {
root /data/fastdfs_data/data;
ngx_fastdfs_module;
}
access_log /data/logs/access_fastdfs.log;
error_log /data/logs/error_fastdfs.log;
listen 80;
server_name fastdfs.hichao.com;
location /group1/M00 {
root /data/fastdfs_data/data;
ngx_fastdfs_module;
}
access_log /data/logs/access_fastdfs.log;
error_log /data/logs/error_fastdfs.log;
}
#----------------------------------------------------------------------------------------------------------------------------------
配置mod_fastdfs.conf
vim /etc/fdfs/mod_fastdfs.conf
#----------------------------------------------------------------------------------------------------------------------------------
connect_timeout=2
network_timeout=30
base_path=/data/fastdfs_data
load_fdfs_parameters_from_tracker=true
storage_sync_file_max_delay = 86400
use_storage_id = false
storage_ids_filename = storage_ids.conf
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
storage_server_port=23000
group_name=group1
url_have_group_name = true
store_path_count=1
store_path0=/data/fastdfs_data
log_level=info
log_filename= /data/logs/mod_fastdfs.log
response_mode=proxy
if_alias_prefix=
#include http.conf
flv_support = true
flv_extension = flv
group_count = 2
[group1]
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/data/fastdfs_data
[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/data/fastdfs_data
#----------------------------------------------------------------------------------------------------------------------------------
#启动并添加开机启动
/usr/local/nginx-1.8.0/sbin/nginx
echo "/usr/local/nginx-1.8.0/sbin/nginx" >> /etc/rc.local
/etc/init.d/fdfs_storaged restart
echo "/etc/init.d/fdfs_storaged restart" >> /etc/rc.local
5：配置group2 (f5 f6配置相同) 不同之处在于group_name需要修改为 group2
nginx location 修改为 group2/M00
storage server 安装 nginx
tar zvfx nginx-1.8.0.tar.gz
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx-1.8.0 \
--with-http_gzip_static_module \
--with-http_gunzip_module \
--add-module=../fastdfs-nginx-module/src
make
make install
cd ..
mkdir -p /usr/local/nginx-1.8.0/conf/vhosts
配置nginx
cp fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/
cp fastdfs-5.05/conf/{anti-steal.jpg,http.conf,mime.types} /etc/fdfs/
touch /data/logs/mod_fastdfs.log
vim nginx.conf
#-----------------------------------------------------------------------------------------------------------------------------------
user root root;
worker_processes 8;
error_log /data/logs/error.log crit;
pid /usr/local/nginx-1.8.0/logs/nginx.pid;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
worker_rlimit_nofile 51200;
events {
use epoll;
worker_connections 51200;
}
http {
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
proxy_headers_hash_max_size 51200;
proxy_headers_hash_bucket_size 6400;
client_header_buffer_size 4k;
large_client_header_buffers 4 32k;
client_max_body_size 1M;
sendfile on;
tcp_nopush on;
keepalive_timeout 300;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 256k;
fastcgi_intercept_errors on;
server_tokens off;
client_body_buffer_size 256k;
send_timeout 3m;
client_header_timeout 3m;
client_body_timeout 3m;
proxy_ignore_client_abort on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
include vhosts/*.conf;
worker_processes 8;
error_log /data/logs/error.log crit;
pid /usr/local/nginx-1.8.0/logs/nginx.pid;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
worker_rlimit_nofile 51200;
events {
use epoll;
worker_connections 51200;
}
http {
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
proxy_headers_hash_max_size 51200;
proxy_headers_hash_bucket_size 6400;
client_header_buffer_size 4k;
large_client_header_buffers 4 32k;
client_max_body_size 1M;
sendfile on;
tcp_nopush on;
keepalive_timeout 300;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 256k;
fastcgi_intercept_errors on;
server_tokens off;
client_body_buffer_size 256k;
send_timeout 3m;
client_header_timeout 3m;
client_body_timeout 3m;
proxy_ignore_client_abort on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
include vhosts/*.conf;
}
#--------------------------------------------------------------------------------------------------------------
vim fastdfs.conf
#--------------------------------------------------------------------------------------------------------------
server {
listen 80;
server_name fastdfs.hichao.com;
location /group2/M00 {
root /data/fastdfs_data/data;
ngx_fastdfs_module;
}
access_log /data/logs/access_fastdfs.log;
error_log /data/logs/error_fastdfs.log;
listen 80;
server_name fastdfs.hichao.com;
location /group2/M00 {
root /data/fastdfs_data/data;
ngx_fastdfs_module;
}
access_log /data/logs/access_fastdfs.log;
error_log /data/logs/error_fastdfs.log;
}
#-------------------------------------------------------------------------------------------------------------
vim storage.conf
#-------------------------------------------------------------------------------------------------------------
disabled=false
group_name=group2
bind_addr=
client_bind=true
port=23000
connect_timeout=30
network_timeout=60
heart_beat_interval=30
stat_report_interval=60
base_path=/data/fastdfs_data
max_connections=256
buff_size = 256KB
accept_threads=1
work_threads=4
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/data/fastdfs_data
subdir_count_per_path=256
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = false
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
group_name=group2
bind_addr=
client_bind=true
port=23000
connect_timeout=30
network_timeout=60
heart_beat_interval=30
stat_report_interval=60
base_path=/data/fastdfs_data
max_connections=256
buff_size = 256KB
accept_threads=1
work_threads=4
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/data/fastdfs_data
subdir_count_per_path=256
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = false
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
http.server_port=80
#----------------------------------------------------------------------------------------------------------
vim mod_fastdfs.conf
#---------------------------------------------------------------------------------------------------------
connect_timeout=2
network_timeout=30
base_path=/data/fastdfs_data
load_fdfs_parameters_from_tracker=true
storage_sync_file_max_delay = 86400
use_storage_id = false
storage_ids_filename = storage_ids.conf
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
storage_server_port=23000
group_name=group2
url_have_group_name = true
store_path_count=1
store_path0=/data/fastdfs_data
log_level=info
log_filename= /data/logs/mod_fastdfs.log
response_mode=proxy
if_alias_prefix=
flv_support = true
flv_extension = flv
group_count = 2
#include http.conf
[group1]
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/data/fastdfs_data
[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
network_timeout=30
base_path=/data/fastdfs_data
load_fdfs_parameters_from_tracker=true
storage_sync_file_max_delay = 86400
use_storage_id = false
storage_ids_filename = storage_ids.conf
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
storage_server_port=23000
group_name=group2
url_have_group_name = true
store_path_count=1
store_path0=/data/fastdfs_data
log_level=info
log_filename= /data/logs/mod_fastdfs.log
response_mode=proxy
if_alias_prefix=
flv_support = true
flv_extension = flv
group_count = 2
#include http.conf
[group1]
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/data/fastdfs_data
[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/data/fastdfs_data
#-------------------------------------------------------------------------------------------------------------------
#启动并添加开机启动
/usr/local/nginx-1.8.0/sbin/nginx
echo "/usr/local/nginx-1.8.0/sbin/nginx" >> /etc/rc.local
/etc/init.d/fdfs_storaged restart
echo "/etc/init.d/fdfs_storaged restart" >> /etc/rc.local
6：配置下载网关路由 f7 f8配置keepalived+nginx双主实现入口层的负责均衡
tar zvfx nginx-1.8.0.tar.gz
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx-1.8.0 \
--with-http_gzip_static_module \
--with-http_gunzip_module
make
make install
cd ..
vim nginx.conf
#----------------------------------------------------------------------------------------------------
user www www;
worker_processes 16;
error_log /data/logs/error.log crit;
pid /usr/local/nginx-1.8.0/nginx.pid;
worker_cpu_affinity
0000000000000001 0000000000000010 0000000000000100 0000000000001000
0000000000010000 0000000000100000 0000000001000000 0000000010000000
0000000100000000 0000001000000000 0000010000000000 0000100000000000
0001000000000000 0010000000000000 0100000000000000 1000000000000000;
worker_rlimit_nofile 51200;
events {
use epoll;
worker_connections 51200;
}
http {
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
proxy_headers_hash_max_size 51200;
proxy_headers_hash_bucket_size 6400;
client_header_buffer_size 64k;
large_client_header_buffers 4 32k;
client_max_body_size 20M;
sendfile on;
tcp_nopush on;
keepalive_timeout 300;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 256k;
server_tokens off;
client_body_buffer_size 256k;
send_timeout 3m;
client_header_timeout 3m;
client_body_timeout 3m;
proxy_ignore_client_abort on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
include vhosts/*.conf;
worker_processes 16;
error_log /data/logs/error.log crit;
pid /usr/local/nginx-1.8.0/nginx.pid;
worker_cpu_affinity
0000000000000001 0000000000000010 0000000000000100 0000000000001000
0000000000010000 0000000000100000 0000000001000000 0000000010000000
0000000100000000 0000001000000000 0000010000000000 0000100000000000
0001000000000000 0010000000000000 0100000000000000 1000000000000000;
worker_rlimit_nofile 51200;
events {
use epoll;
worker_connections 51200;
}
http {
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
proxy_headers_hash_max_size 51200;
proxy_headers_hash_bucket_size 6400;
client_header_buffer_size 64k;
large_client_header_buffers 4 32k;
client_max_body_size 20M;
sendfile on;
tcp_nopush on;
keepalive_timeout 300;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 256k;
server_tokens off;
client_body_buffer_size 256k;
send_timeout 3m;
client_header_timeout 3m;
client_body_timeout 3m;
proxy_ignore_client_abort on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
include vhosts/*.conf;
}
#---------------------------------------------------------------------------------------------------
mkdir -p /usr/local/nginx-1.8.0/conf/vhosts
vim fastdfs.hichao.com.conf
#--------------------------------------------------------------------------------------------------
server {
listen 80;
server_name fastdfs.hichao.com;
access_log /data/logs/nginx/fastdfs.hichao.com.access.log;
error_log /data/logs/nginx/fastdfs.hichao.com.error.log;
location /group1 {
proxy_pass_header Server;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Scheme $scheme;
proxy_pass http://group1;
}
location /group2 {
proxy_pass_header Server;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Scheme $scheme;
proxy_pass http://group2;
}
listen 80;
server_name fastdfs.hichao.com;
access_log /data/logs/nginx/fastdfs.hichao.com.access.log;
error_log /data/logs/nginx/fastdfs.hichao.com.error.log;
location /group1 {
proxy_pass_header Server;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Scheme $scheme;
proxy_pass http://group1;
}
location /group2 {
proxy_pass_header Server;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Scheme $scheme;
proxy_pass http://group2;
}
}
#--------------------------------------------------------------------------------------------------
vim upstream.conf
#-------------------------------------------------------------------------------------------------
upstream group1 {
server 192.168.1.81:80;
server 192.168.1.82:80;
}
upstream group2 {
server 192.168.1.83:80;
server 192.168.1.84:80;
server 192.168.1.82:80;
}
upstream group2 {
server 192.168.1.83:80;
server 192.168.1.84:80;
}
#-------------------------------------------------------------------------------------------------
keepalived 部署略。
至此 基本的架构部署完毕！
7：在线扩容 增加group3 (f7 f8配置相同)
1：关闭相关服务(所有机器)
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
chkconfig iptables off
mkdir -p /data/fastdfs_data
yum install -y wget
/bin/mv /etc/yum.repos.d/* /tmp/
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.163.com/.help/CentOS6-Base-163.repo
yum makecache
yum install -y gettext gettext-devel libXft libXft-devel libXpm libXpm-devel automake autoconf libXtst-devel gtk+-devel gcc gcc-c++ zlib-devel libpng-devel gtk2-devel glib-devel pcre* gcc git
2：安装libfastcommon and fastdfs(所有机器)
mkdir fastdfs
cd fastdfs
cd libfastcommon/
./make.sh
./make.sh install
cd ..
tar zvfx V5.05.tar.gz
cd fastdfs-5.05/
sed -i 's#/usr/local/bin/#/usr/bin/#g' init.d/fdfs_storaged
sed -i 's#/usr/local/bin/#/usr/bin/#g' init.d/fdfs_trackerd
./make.sh
./make.sh install
cd ..
cp /etc/fdfs/storage.conf.sample /etc/fdfs/storage.conf
vim /etc/fdfs/storage.conf
#---------------------------------------------------------------------------------------------------------------------------------
disabled=false
group_name=group3
bind_addr=
client_bind=true
port=23000
connect_timeout=30
network_timeout=60
heart_beat_interval=30
stat_report_interval=60
base_path=/data/fastdfs_data
max_connections=256
buff_size = 256KB
accept_threads=1
work_threads=4
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/data/fastdfs_data
subdir_count_per_path=256
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = false
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
client_bind=true
port=23000
connect_timeout=30
network_timeout=60
heart_beat_interval=30
stat_report_interval=60
base_path=/data/fastdfs_data
max_connections=256
buff_size = 256KB
accept_threads=1
work_threads=4
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/data/fastdfs_data
subdir_count_per_path=256
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = false
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
http.server_port=80
#---------------------------------------------------------------------------------------------------------------------------------
storage server 安装 nginx
tar zvfx nginx-1.8.0.tar.gz
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx-1.8.0 \
--with-http_gzip_static_module \
--with-http_gunzip_module \
--add-module=../fastdfs-nginx-module/src
make
make install
cd ..
配置nginx
cp fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/
cp fastdfs-5.05/conf/{anti-steal.jpg,http.conf,mime.types} /etc/fdfs/
touch /data/logs/mod_fastdfs.log
vim nginx.conf
#-----------------------------------------------------------------------------------------------------------------------------------
user root root;
worker_processes 8;
worker_processes 8;
error_log /data/logs/error.log crit;
pid /usr/local/nginx-1.8.0/logs/nginx.pid;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
worker_rlimit_nofile 51200;
events {
use epoll;
worker_connections 51200;
}
http {
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
proxy_headers_hash_max_size 51200;
proxy_headers_hash_bucket_size 6400;
client_header_buffer_size 4k;
large_client_header_buffers 4 32k;
client_max_body_size 1M;
sendfile on;
tcp_nopush on;
keepalive_timeout 300;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 256k;
fastcgi_intercept_errors on;
server_tokens off;
client_body_buffer_size 256k;
send_timeout 3m;
client_header_timeout 3m;
client_body_timeout 3m;
proxy_ignore_client_abort on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
include vhosts/*.conf;
}
#-----------------------------------------------------------------------------------------------------------------------------------
mkdir -p /usr/local/nginx-1.8.0/conf/vhosts
vim /usr/local/nginx-1.8.0/conf/vhosts/fastdfs.conf
#----------------------------------------------------------------------------------------------------------------------------------
server {
listen 80;
server_name fastdfs.hichao.com;
location /group3/M00 {
root /data/fastdfs_data/data;
ngx_fastdfs_module;
}
access_log /data/logs/access_fastdfs.log;
error_log /data/logs/error_fastdfs.log;
ngx_fastdfs_module;
}
access_log /data/logs/access_fastdfs.log;
error_log /data/logs/error_fastdfs.log;
}
#----------------------------------------------------------------------------------------------------------------------------------
配置mod_fastdfs.conf
vim /etc/fdfs/mod_fastdfs.conf
#----------------------------------------------------------------------------------------------------------------------------------
connect_timeout=2
network_timeout=30
base_path=/data/fastdfs_data
load_fdfs_parameters_from_tracker=true
storage_sync_file_max_delay = 86400
use_storage_id = false
storage_ids_filename = storage_ids.conf
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
storage_server_port=23000
group_name=group3
url_have_group_name = true
store_path_count=1
store_path0=/data/fastdfs_data
log_level=info
log_filename= /data/logs/mod_fastdfs.log
response_mode=proxy
if_alias_prefix=
#include http.conf
flv_support = true
flv_extension = flv
group_count = 3
[group1]
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/data/fastdfs_data
[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/data/fastdfs_data
[group3]
group_name=group3
storage_server_port=23000
store_path_count=1
store_path0=/data/fastdfs_data
#----------------------------------------------------------------------------------------------------------------------------------
#启动并添加开机启动
/usr/local/nginx-1.8.0/sbin/nginx
echo "/usr/local/nginx-1.8.0/sbin/nginx" >> /etc/rc.local
/etc/init.d/fdfs_storaged restart
echo "/etc/init.d/fdfs_storaged restart" >> /etc/rc.local
在线扩容完毕
测试：
vim /etc/fdfs/client.conf
#--------------------------------------------
connect_timeout=30
network_timeout=60
base_path=/tmp
tracker_server=192.168.1.79:22122
tracker_server=192.168.1.80:22122
log_level=info
use_connection_pool = false
connection_pool_max_idle_time = 3600
load_fdfs_parameters_from_tracker=false
use_storage_id = false
storage_ids_filename = storage_ids.conf
http.tracker_server_port=80
#--------------------------------------------
[root@f1 ~]# fdfs_upload_file /etc/fdfs/client.conf a.log
group2/M00/00/00/wKgBVFamz7eARyk5AAAAEIAIMqE338.log
[root@f1 ~]# fdfs_download_file /etc/fdfs/client.conf group2/M00/00/00/wKgBVFamz7eARyk5AAAAEIAIMqE338.log
[root@f1 ~]# cat wKgBVFamz7eARyk5AAAAEIAIMqE338.log
测试第一行
[root@f1 ~]# fdfs_file_info /etc/fdfs/client.conf group2/M00/00/00/wKgBVFamz7eARyk5AAAAEIAIMqE338.log
source storage id: 0
source ip address: 192.168.1.84
file create timestamp: 2016-01-26 09:45:27
file size: 16
file crc32: 2148020897 (0x800832A1)
删除文件
fdfs_delete_file /etc/fdfs/client.conf group2/M00/00/00/wKgBVFam0X6ARLKXAAAAEIAIMqE826.log
查看集群状态：
fdfs_monitor /etc/fdfs/client.conf
查看文件校验
[root@f1 ~]# fdfs_crc32 /etc/fdfs/client.conf group2/M00/00/00/wKgBVFam0X6ARLKXAAAAEIAIMqE826.log
795743293
将离线的storage剔除组：
fdfs_monitor /etc/fdfs/client.conf delete group3 192.168.1.86
