ubuntu系统每天凌晨定时上传redis 备份数据到阿里云OSS上

1.压缩备份脚本

1.1 代码如下

#!/bin/bash

# redis_backup_compress.sh
# 设置变量
BACKUP_DIR="/data/redis/backup"
REDIS_DIR="/var/lib/redis"
DATE=$(date +%Y%m%d_%H%M%S)
BACKUP_FILE="redis_backup_${DATE}.rdb"
COMPRESSED_FILE="redis_backup_${DATE}.tar.gz"
OSS_BUCKET="your-bucket-name"
OSS_PATH="redis/backup/"

# 阿里云访问凭证
ACCESS_KEY="your_access_key"
ACCESS_SECRET="your_access_secret"
ENDPOINT="oss-cn-beijing.aliyuncs.com"

# 创建备份目录
mkdir -p ${BACKUP_DIR}

# 记录日志函数
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" >> "${BACKUP_DIR}/backup.log"
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1"
}

# 检查并安装必要工具
check_tools() {
    for tool in ossutil64 tar pigz; do
        if ! command -v $tool &> /dev/null; then
            log "Installing $tool..."
            case $tool in
                "ossutil64")
                    wget http://gosspublic.alicdn.com/ossutil/1.7.7/ossutil64 -O /usr/local/bin/ossutil64
                    chmod 755 /usr/local/bin/ossutil64
                    ;;
                "pigz")
                    apt-get update && apt-get install -y pigz
                    ;;
                "tar")
                    apt-get update && apt-get install -y tar
                    ;;
            esac
        fi
    done
}

# 配置ossutil
configure_ossutil() {
    if [ ! -f ~/.ossutilconfig ]; then
        log "Configuring ossutil..."
        ossutil64 config -e ${ENDPOINT} -i ${ACCESS_KEY} -k ${ACCESS_SECRET}
    fi
}

# 备份Redis
backup_redis() {
    log "Starting Redis backup..."
    
    # 复制当前的dump.rdb文件
    if cp ${REDIS_DIR}/dump.rdb ${BACKUP_DIR}/${BACKUP_FILE}; then
        log "Redis backup created: ${BACKUP_FILE}"
        
        # 使用tar和pigz进行多线程压缩
        log "Compressing backup file..."
        cd ${BACKUP_DIR}
        if tar cf - ${BACKUP_FILE} | pigz -9 -p 4 > ${COMPRESSED_FILE}; then
            log "Backup compressed successfully: ${COMPRESSED_FILE}"
            # 删除原始备份文件
            rm ${BACKUP_FILE}
            return 0
        else
            log "Failed to compress backup"
            return 1
        fi
    else
        log "Failed to create Redis backup"
        return 1
    fi
}

# 计算文件MD5
calculate_md5() {
    local file=$1
    md5sum ${file} | cut -d' ' -f1
}

# 上传到阿里云OSS
upload_to_oss() {
    log "Uploading compressed backup to OSS..."
    
    local file_path="${BACKUP_DIR}/${COMPRESSED_FILE}"
    local md5_before=$(calculate_md5 ${file_path})
    
    # 使用ossutil的断点续传功能上传
    if ossutil64 cp -u ${file_path} \
        oss://${OSS_BUCKET}/${OSS_PATH}${COMPRESSED_FILE} \
        --parallel 4 \
        --checkpoint-dir ${BACKUP_DIR}/.checkpoint_dir; then
        
        # 验证上传
        ossutil64 cp oss://${OSS_BUCKET}/${OSS_PATH}${COMPRESSED_FILE} \
            ${BACKUP_DIR}/verify_${COMPRESSED_FILE} --parallel 4
            
        local md5_after=$(calculate_md5 ${BACKUP_DIR}/verify_${COMPRESSED_FILE})
        
        if [ "${md5_before}" = "${md5_after}" ]; then
            log "Backup uploaded and verified successfully"
            rm ${BACKUP_DIR}/verify_${COMPRESSED_FILE}
            return 0
        else
            log "Upload verification failed"
            return 1
        fi
    else
        log "Failed to upload backup to OSS"
        return 1
    fi
}

# 清理旧备份
cleanup_old_backups() {
    log "Cleaning up old backups..."
    
    # 保留本地最近3天的备份
    find ${BACKUP_DIR} -name "redis_backup_*.tar.gz" -type f -mtime +3 -delete
    
    # 清理OSS上30天前的备份
    ossutil64 rm oss://${OSS_BUCKET}/${OSS_PATH} --recursive --force \
        --include "redis_backup_*.tar.gz" \
        --timerange "30daysago"
        
    # 清理检查点目录
    find ${BACKUP_DIR}/.checkpoint_dir -mtime +1 -type f -delete
}

# 监控备份大小
monitor_backup_size() {
    local file_size=$(du -h ${BACKUP_DIR}/${COMPRESSED_FILE} | cut -f1)
    local compression_ratio=$(echo "scale=2; $(stat -f %z ${REDIS_DIR}/dump.rdb) / \
        $(stat -f %z ${BACKUP_DIR}/${COMPRESSED_FILE})" | bc)
    
    log "Backup size: ${file_size}, Compression ratio: ${compression_ratio}x"
    
    # 如果备份文件异常大，发送告警
    if [ $(stat -f %z ${BACKUP_DIR}/${COMPRESSED_FILE}) -gt 1073741824 ]; then # 1GB
        send_notification "WARNING" "Backup file is larger than 1GB: ${file_size}"
    fi
}

# 发送通知
send_notification() {
    local status=$1
    local message=$2
    
    # 这里可以集成你的通知方式（邮件、钉钉、企业微信等）
    if [ -n "${NOTIFICATION_WEBHOOK}" ]; then
        curl -X POST ${NOTIFICATION_WEBHOOK} \
            -H 'Content-Type: application/json' \
            -d "{\"status\":\"${status}\",\"message\":\"${message}\"}"
    fi
}

# 主函数
main() {
    log "=== Starting backup process ==="
    
    # 检查并安装工具
    check_tools
    configure_ossutil
    
    # 执行备份
    if backup_redis; then
        monitor_backup_size
        
        # 上传到OSS
        if upload_to_oss; then
            log "Backup process completed successfully"
            send_notification "SUCCESS" "Redis backup completed and uploaded to OSS"
            
            # 清理旧备份
            cleanup_old_backups
        else
            log "Backup process failed during upload"
            send_notification "FAILED" "Failed to upload Redis backup to OSS"
            exit 1
        fi
    else
        log "Backup process failed during backup creation"
        send_notification "FAILED" "Failed to create Redis backup"
        exit 1
    fi
    
    log "=== Backup process finished ==="
}

# 执行主函数
main

2. 解压恢复脚本

2.1 代码如下

#!/bin/bash

# redis_restore_decompress.sh
BACKUP_DIR="/data/redis/backup"
REDIS_DIR="/var/lib/redis"
OSS_BUCKET="your-bucket-name"
OSS_PATH="redis/backup/"

# 参数检查
if [ $# -ne 1 ]; then
    echo "Usage: $0 <compressed_backup_file_name>"
    exit 1
fi

COMPRESSED_FILE=$1
BACKUP_FILE="${COMPRESSED_FILE%.tar.gz}.rdb"

# 从OSS下载压缩的备份
download_from_oss() {
    echo "Downloading compressed backup from OSS..."
    ossutil64 cp oss://${OSS_BUCKET}/${OSS_PATH}${COMPRESSED_FILE} \
        ${BACKUP_DIR}/${COMPRESSED_FILE} --parallel 4
}

# 解压备份
decompress_backup() {
    echo "Decompressing backup..."
    cd ${BACKUP_DIR}
    pigz -d < ${COMPRESSED_FILE} | tar xf -
    
    if [ -f "${BACKUP_DIR}/${BACKUP_FILE}" ]; then
        echo "Backup decompressed successfully"
        return 0
    else
        echo "Failed to decompress backup"
        return 1
    fi
}

# 恢复备份
restore_backup() {
    echo "Stopping Redis server..."
    sudo service redis stop
    
    echo "Restoring backup..."
    sudo cp ${BACKUP_DIR}/${BACKUP_FILE} ${REDIS_DIR}/dump.rdb
    sudo chown redis:redis ${REDIS_DIR}/dump.rdb
    
    echo "Starting Redis server..."
    sudo service redis start
    
    # 验证Redis是否正常启动
    sleep 5
    if redis-cli ping | grep -q "PONG"; then
        echo "Redis server started successfully"
        return 0
    else
        echo "Failed to start Redis server"
        return 1
    fi
}

# 清理临时文件
cleanup() {
    rm -f ${BACKUP_DIR}/${COMPRESSED_FILE}
    rm -f ${BACKUP_DIR}/${BACKUP_FILE}
}

# 主函数
main() {
    echo "=== Starting restore process ==="
    
    # 下载压缩的备份
    download_from_oss
    
    # 解压备份
    if decompress_backup; then
        # 恢复备份
        if restore_backup; then
            echo "=== Restore process completed successfully ==="
            cleanup
        else
            echo "=== Restore process failed ==="
            exit 1
        fi
    else
        echo "=== Restore process failed during decompression ==="
        exit 1
    fi
}

main

3.使用说明

3.1 安装必要工具：

sudo apt-get update
sudo apt-get install -y wget pigz tar bc mailutils

3.2 设置执行权限：

sudo chmod +x redis_backup_compress.sh
sudo chmod +x redis_restore_decompress.sh

3.3 配置定时任务：

# 编辑crontab
crontab -e

# 添加以下内容（每天凌晨2点执行）
0 2 * * * /path/to/redis_backup_compress.sh

3.4 手动测试：

# 执行备份
sudo ./redis_backup_compress.sh

# 查看压缩文件大小
ls -lh /data/redis/backup/

# 恢复备份
sudo ./redis_restore_decompress.sh redis_backup_20240101_020000.tar.gz