两个主机上的Docker容器怎么实现连接
背景:
ehcache缓存同步,之前是不同主机上部署服务,后面改成容器了
容器默认创建使用的是桥接网络,所以两台主机上的容器就上做了主机端口映射,实际缓存同步饿时候也是访问不了的,报错,ehcache里面配置的是主机ip,报错的是容器ip
... 8 common frames omitted
2024-08-29 16:37:41.735 [] [Replication Thread] WARN n.s.e.d.RMIAsynchronousCacheReplicator - Unable to send message to remote peer. Message was: Connection refused to host: 172.21.0.2; nested exception is:
java.net.ConnectException: Connection timed out (Connection timed out)
java.rmi.ConnectException: Connection refused to host: 172.21.0.2; nested exception is:
java.net.ConnectException: Connection timed out (Connection timed out)
at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:623)
at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:216)
at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:202)
at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:132)
at net.sf.ehcache.distribution.RMICachePeer_Stub.send(Unknown Source)
at net.sf.ehcache.distribution.RMIAsynchronousCacheReplicator.writeReplicationQueue(RMIAsynchronousCacheReplicator.java:314)
at net.sf.ehcache.distribution.RMIAsynchronousCacheReplicator.replicationThreadMain(RMIAsynchronousCacheReplicator.java:127)
at net.sf.ehcache.distribution.RMIAsynchronousCacheReplicator.access$000(RMIAsynchronousCacheReplicator.java:58)
at net.sf.ehcache.distribution.RMIAsynchronousCacheReplicator$ReplicationThread.run(RMIAsynchronousCacheReplicator.java:389)
Caused by: java.net.ConnectException: Connection timed out (Connection timed out)
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:607)
at java.net.Socket.connect(Socket.java:556)
at java.net.Socket.<init>(Socket.java:452)
at java.net.Socket.<init>(Socket.java:229)
at sun.rmi.transport.proxy.RMIDirectSocketFactory.createSocket(RMIDirectSocketFactory.java:40)
at sun.rmi.transport.proxy.RMIMasterSocketFactory.createSocket(RMIMasterSocketFactory.java:148)
at net.sf.ehcache.distribution.ConfigurableRMIClientSocketFactory.createSocket(ConfigurableRMIClientSocketFactory.java:70)
at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:617)
... 8 common frames omitted
Default Locale: zh
ehcache.xml配置:要用主机ip哦
<ehcache>
<diskStore path="java.io.tmpdir" />
<defaultCache maxElementsInMemory="10000" eternal="false"
timeToIdleSeconds="4" timeToLiveSeconds="4" overflowToDisk="true" />
<cache name="daoCache" maxElementsInMemory="3000" eternal="false"
timeToIdleSeconds="14400" timeToLiveSeconds="86400"
overflowToDisk="true" >
<cacheEventListenerFactory class="net.sf.ehcache.distribution.RMICacheReplicatorFactory" />
</cache>
<!-- localhost 用主机ip代替 -->
<cacheManagerPeerListenerFactory class="net.sf.ehcache.distribution.RMICacheManagerPeerListenerFactory"
properties="hostName=localhost, port=40001,socketTimeoutMillis=2000" />
<cacheManagerPeerProviderFactory class="net.sf.ehcache.distribution.RMICacheManagerPeerProviderFactory"
properties="peerDiscovery=manual,rmiUrls=//10.0.192.29:40001/daoCache" />
<!-- jbpm cache -->
<Cache name="org.jbpm.graph.def.ProcessDefinition" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Node" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Transition" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Event" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Action" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.SuperState" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.ExceptionHandler" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.instantiation.Delegation" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.taskmgmt.def.Task" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.module.def.ModuleDefinition" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.context.def.VariableAccess" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.taskmgmt.def.TaskController" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.node.ProcessState.variableAccesses" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.taskmgmt.def.Task.exceptionHandlers" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Node.exceptionHandlers" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Transition.events" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.taskmgmt.def.TaskMgmtDefinition.tasks" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.ProcessDefinition.events" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.taskmgmt.def.Swimlane.tasks" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Node.leavingTransitions" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.taskmgmt.def.TaskController.variableAccesses" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.ProcessDefinition.exceptionHandlers" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.node.Decision.decisionConditions" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.file.def.FileDefinition.processFiles" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.ProcessDefinition.actions" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Transition.exceptionHandlers" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.ProcessDefinition.nodes" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Node.arrivingTransitions" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.node.TaskNode.tasks" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.ProcessDefinition.definitions" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.taskmgmt.def.Task.events" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Node.events" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.taskmgmt.def.TaskMgmtDefinition.swimlanes" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.SuperState.nodes" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.action.Script.variableAccesses" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.ExceptionHandler.actions" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
<Cache name="org.jbpm.graph.def.Event.actions" maxElementsInMemory="100000" eternal="true" overflowToDisk="false" diskPersistent="false"/>
</ehcache>
方案一、容器网络模式改成host。
增加network_mode: host。这是方式用于测试可以,实际应该最好不跟主机绑定。
修改后docker-compose配置
version: "3"
services:
you-service:
image: openjdk:8-jdk
container_name: you-service
restart: always
network_mode: host
#ports:
# - 8080:8080
command: java -jar /opt/you-service.jar
volumes:
- /data/services/you-service-path/:/opt/
- /data/log/you-path/:/logs/
environment:
- TZ=Asia/Shanghai
- SERVICE_HOST=10.0.254.60
- server.port=8080
- NACOS_NAMESPACE=${NACOS_NAMESPACE}
- NACOS_ADDR=10.0.250.106:8848
- JAVA_OPTS=-Xmx512m -XX:G1ConcRefinementThreads=4 -XX:MaxDirectMemorySize=1G
注意:增加网络主机模式后需要删除端口映射,否则会报错
ERROR: for sms-service-hb1 "host" network_mode is incompatible with port_bindings
方案二:手动配置跨主机桥接网络。
正向配置操作:
步凑如下:
1、每个主机上创建桥接网络
docker network create --driver bridge my-net
2、配置ip。为每个主机上的 Docker 桥接网络分配一个唯一的 IP 地址。假设你有两个主机 host1 和 host2,你可以分别为它们的 docker0 网桥配置不同的 IP 地址
# 在 host1 上配置
sudo ip addr add 192.168.1.1/24 dev docker0
# 在 host2 上配置
sudo ip addr add 192.168.1.2/24 dev docker0
3、配置静态路由
在每台主机上配置静态路由,使得一台主机上的 Docker 桥接网络可以到达另一台主机上的 Docker 桥接网络。例如,从 host1 到 host2 的路由配置如下:
# 在 host1 上配置路由
sudo route add -net 192.168.1.0 netmask 255.255.255.0 gw 192.168.1.2 dev docker0
# 在 host2 上配置路由
sudo route add -net 192.168.1.0 netmask 255.255.255.0 gw 192.168.1.1 dev docker0
4、配置容器网络
version: "3"
services:
you-service:
image: openjdk:8-jdk
container_name: you-service
restart: always
#network_mode: host
networks:
- my-net
ports:
- 8080:8080
command: java -jar /opt/you-service.jar
volumes:
- /data/services/you-service-path/:/opt/
- /data/log/you-path/:/logs/
environment:
- TZ=Asia/Shanghai
- SERVICE_HOST=10.0.254.60
- server.port=8080
- NACOS_NAMESPACE=${NACOS_NAMESPACE}
- NACOS_ADDR=10.0.250.106:8848
- JAVA_OPTS=-Xmx512m -XX:G1ConcRefinementThreads=4 -XX:MaxDirectMemorySize=1G
networks:
my-net:
external: true
5、创建并启动容器
docker-compose up
反向删除操作:
步凑如下:
1、停止容器并删除
docker-compose down
2、删除静态路由
route -n
route del -net 192.168.1.0 netmask 255.255.255.0 gw 192.168.1.1 dev docker0
3、删除ip
ip addr show docker0
ip addr del 192.168.1.2/24 dev docker0
4、删除网络
docker network ls
docker network rm my-net
方案三:配置覆盖网络Overlay,但是要用到Docker Swarm。
如果你愿意使用 Docker Swarm,可以创建一个跨主机的 Overlay 网络。这允许容器在不同的主机上进行通信,就像它们在一个网络上一样。
未试验
方案四、使用第三方网络解决方案