当前位置：首页 > article >正文

python+docker实现分布式存储的demo

article 2025/1/20 3:48:55

test.py代码

#test.py
from flask import Flask, request, jsonify
import requests
import sys
import threading

app = Flask(__name__)

# 存储数据
data_store = {}

# 节点列表，通过环境变量传入
nodes = []
current_node = None

@app.route('/set', methods=['POST'])
def set_value():
    key = request.json.get('key')
    value = request.json.get('value')
    data_store[key] = value
    # 同步到其他节点
    for node in nodes:
        if node != current_node:
            try:
                requests.post(f'http://{node}/sync', json={'key': key, 'value': value})
            except requests.exceptions.RequestException:
                pass  # 可以添加更好的错误处理
    return jsonify({'status': 'success'}), 200

@app.route('/get', methods=['GET'])
def get_value():
    key = request.args.get('key')
    value = data_store.get(key, None)
    return jsonify({'value': value}), 200

@app.route('/sync', methods=['POST'])
def sync():
    key = request.json.get('key')
    value = request.json.get('value')
    data_store[key] = value
    return jsonify({'status': 'synced'}), 200

@app.route('/nodes', methods=['GET'])
def get_nodes():
    return jsonify({'nodes': nodes}), 200

def run_app(port):
    app.run(host='0.0.0.0', port=port)

if __name__ == '__main__':
    # 从环境变量获取节点信息
    current_node = sys.argv[1]  # 例如：'node1:5000'
    nodes = sys.argv[2].split(',')  # 例如：'node1:5000,node2:5001,node3:5002'
    port = int(current_node.split(':')[1])
    run_app(port)

Dockerfile:

# Dockerfile
FROM python:3.9-slim

WORKDIR /app

COPY test.py /app/test.py
COPY requirements.txt /app/requirements.txt

RUN pip install --no-cache-dir -r requirements.txt

EXPOSE 5000

CMD ["python", "test.py"]

docker-compose.yml

version: '3'

services:
  node1:
    build: .
    container_name: node1
    ports:
      - "5000:5000"
    environment:
      - NODE_NAME=node1
    command: ["python", "test.py", "node1:5000", "node1:5000,node2:5000,node3:5000"]
    networks:
      - dnetwork

  node2:
    build: .
    container_name: node2
    ports:
      - "5001:5000"
    environment:
      - NODE_NAME=node2
    command: ["python", "test.py", "node2:5000", "node1:5000,node2:5000,node3:5000"]
    networks:
      - dnetwork

  node3:
    build: .
    container_name: node3
    ports:
      - "5002:5000"
    environment:
      - NODE_NAME=node3
    command: ["python", "test.py", "node3:5000", "node1:5000,node2:5000,node3:5000"]
    networks:
      - dnetwork

networks:
  dnetwork:
    driver: bridge

requirements.txt

Flask
requests

上面的demo是用docker-compose启动三个容器（这三个容器的镜像都是由同一个Dockerfile实现的），然后实现的功能就是，比如我往容器一存东西，然后可以在容器2，容器3查看到

curl测试

curl -X POST -H "Content-Type: application/json" \
     -d '{"key": "foo", "value": "bar"}' \
     http://localhost:5000/set



curl http://localhost:5002/get?key=foo

然后我们以后还可以扩展，将docker-compose启动的镜像换为三个不同的镜像来完成你的项目需求（例如分布式机器学习，训练模型等等），整体的demo是搭起来了，后面你的具体需求就需要你自己修改实现了