diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/AddressUtils.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/AddressUtils.java index 99110c120..e4fc8fd79 100644 --- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/AddressUtils.java +++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/AddressUtils.java @@ -20,11 +20,13 @@ import com.alibaba.polardbx.common.utils.logger.LoggerFactory; import com.google.common.base.Splitter; import lombok.Data; +import org.apache.commons.lang.StringUtils; import java.io.IOException; import java.net.InetAddress; import java.net.NetworkInterface; import java.net.ServerSocket; +import java.net.SocketException; import java.util.Enumeration; import java.util.List; import java.util.regex.Pattern; @@ -88,6 +90,10 @@ private static boolean isValidHostAddress(InetAddress address) { return false; } String name = address.getHostAddress(); + return isIpValid(name); + } + + private static boolean isIpValid(String name) { return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name) .matches()); } @@ -148,6 +154,7 @@ public static String getPaxosAddressByStorageAddress(String storageAddr) { } public static InetAddress localAddress = null; + public final static String POD_IP = "POD_IP"; public static InetAddress getHostAddress() { if (localAddress != null) { @@ -155,6 +162,22 @@ public static InetAddress getHostAddress() { } try { + String podIp = System.getenv(POD_IP); + if (!StringUtils.isBlank(podIp)) { + podIp = podIp.trim(); + if (!isIpValid(podIp)) { + logger.error("pod ip not empty but not valid, pod ip is: " + podIp); + } else { + localAddress = getMatchedAddress(podIp); + if (localAddress != null) { + logger.info("get matched address from pod ip: " + podIp); + return localAddress; + } else { + // local address should not null + logger.error("Failed to matched pod ip to address."); + } + } + } localAddress = InetAddress.getLocalHost(); if (isValidHostAddress(localAddress)) { return localAddress; @@ -164,7 +187,7 @@ public static InetAddress getHostAddress() { + e.getMessage()); } try { - Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + Enumeration interfaces = getNetInterface(); if (interfaces != null) { while (interfaces.hasMoreElements()) { try { @@ -194,4 +217,53 @@ public static InetAddress getHostAddress() { logger.error("Could not get local host ip address, will use 127.0.0.1 instead."); return localAddress; } + + public static InetAddress getMatchedAddress(String podIp) { + InetAddress matchedHost = null; + try { + matchedHost = tryMatchFromLocalHost(podIp); + if (matchedHost != null) { + return matchedHost; + } + + matchedHost = tryMatchFromAllNet(podIp, getNetInterface()); + } catch (Throwable e) { + logger.error("Failed to matched pod ip to address. cause: " + e.getMessage()); + } + return matchedHost; + } + + public static InetAddress tryMatchFromLocalHost(String podIp) { + try { + InetAddress localHost = InetAddress.getLocalHost(); + if (isValidHostAddress(localHost) && podIp.equalsIgnoreCase(localHost.getHostAddress())) { + return localHost; + } + } catch (Throwable e) { + // just ignore this + } + return null; + } + + public static InetAddress tryMatchFromAllNet(String podIp, Enumeration interfaces) { + if (interfaces != null) { + while (interfaces.hasMoreElements()) { + NetworkInterface network = interfaces.nextElement(); + Enumeration addresses = network.getInetAddresses(); + if (addresses != null) { + while (addresses.hasMoreElements()) { + InetAddress address = addresses.nextElement(); + if (isValidHostAddress(address) && podIp.equalsIgnoreCase(address.getHostAddress())) { + return address; + } + } + } + } + } + return null; + } + + public static Enumeration getNetInterface() throws SocketException { + return NetworkInterface.getNetworkInterfaces(); + } } diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/AddressUtilsTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/AddressUtilsTest.java new file mode 100644 index 000000000..e403a7e67 --- /dev/null +++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/AddressUtilsTest.java @@ -0,0 +1,77 @@ +package com.alibaba.polardbx.common; + +import com.alibaba.polardbx.common.utils.AddressUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class AddressUtilsTest { + static final String MOCK_IP = "30.221.112.137"; + static final String SERIALIZED_INET_ADDRESS_PATH = "serialize/inet_address.txt"; + + @Test + public void testMatchPodIpFromLocalHost() { + Assert.assertNull(AddressUtils.tryMatchFromLocalHost(MOCK_IP)); + } + + @Test + public void testMatchPodIpFromAllNet() + throws IOException, ClassNotFoundException, InvocationTargetException, NoSuchMethodException, + InstantiationException, IllegalAccessException { + + Assert.assertNull(AddressUtils.tryMatchFromAllNet(MOCK_IP, AddressUtils.getNetInterface())); + + // test mock interface + InetAddress inetAddress = AddressUtils.tryMatchFromAllNet( + MOCK_IP, Collections.enumeration(generateMockInterfaces())); + Assert.assertEquals(inetAddress.getHostAddress(), MOCK_IP); + } + + @Test + public void checkNotMatchedAddress() { + Assert.assertNull(AddressUtils.getMatchedAddress("127.0.0.1")); + Assert.assertNull(AddressUtils.getMatchedAddress("0.0.0.0")); + Assert.assertNull(AddressUtils.getMatchedAddress("123.xx.23.1")); + } + + private List generateMockInterfaces() + throws IOException, InvocationTargetException, NoSuchMethodException, InstantiationException, + IllegalAccessException, ClassNotFoundException { + List interfaceList = + new ArrayList<>(Collections.list(NetworkInterface.getNetworkInterfaces())); + NetworkInterface mockNetInterface = mockNetInterface(); + interfaceList.add(mockNetInterface); + return interfaceList; + } + + private NetworkInterface mockNetInterface() + throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, + IOException, ClassNotFoundException { + Class clazz = NetworkInterface.class; + Constructor constructor = clazz.getDeclaredConstructor(String.class, int.class, InetAddress[].class); + constructor.setAccessible(true); + NetworkInterface mockInterface = (NetworkInterface) constructor.newInstance("mock_ens", 1001, + new InetAddress[] {getMockInetAddress()}); + return mockInterface; + } + + private InetAddress getMockInetAddress() throws IOException, ClassNotFoundException { + String path = this.getClass().getClassLoader().getResource(SERIALIZED_INET_ADDRESS_PATH).getPath(); + try (FileInputStream fileIn = new FileInputStream(path); + ObjectInputStream objIn = new ObjectInputStream(fileIn)) { + + InetAddress inetAddress = (InetAddress) objIn.readObject(); + return inetAddress; + } + } +} diff --git a/polardbx-common/src/test/resources/serialize/inet_address.txt b/polardbx-common/src/test/resources/serialize/inet_address.txt new file mode 100644 index 000000000..03692358a Binary files /dev/null and b/polardbx-common/src/test/resources/serialize/inet_address.txt differ diff --git a/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/node/GmsNodeManager.java b/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/node/GmsNodeManager.java index 28580c750..15d629793 100644 --- a/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/node/GmsNodeManager.java +++ b/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/node/GmsNodeManager.java @@ -45,6 +45,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; public class GmsNodeManager extends AbstractLifecycle { @@ -323,6 +324,13 @@ private void loadRedundantNodes() { return gmsNode1.instId.compareTo(gmsNode2.instId); }); this.currentIndex = allNodes.indexOf(GmsNodeManager.getInstance().getLocalNode()); + if (currentIndex == -1) { + LOGGER.error(String.format( + "local node not found from allNodes, local node is %s, while all node is %s", + GmsNodeManager.getInstance().getLocalNode(), + allNodes.stream().map(node -> node.toString()).collect(Collectors.joining(","))) + ); + } } private GmsNode buildNode(ServerInfoRecord record, int uniqueId) { diff --git a/polardbx-server/src/main/conf/logback.xml b/polardbx-server/src/main/conf/logback.xml index 15cdc9029..a30e5aa21 100644 --- a/polardbx-server/src/main/conf/logback.xml +++ b/polardbx-server/src/main/conf/logback.xml @@ -991,6 +991,9 @@ + + + diff --git a/polardbx-server/src/main/resources/logback.xml b/polardbx-server/src/main/resources/logback.xml index a378b17ee..348b6517c 100644 --- a/polardbx-server/src/main/resources/logback.xml +++ b/polardbx-server/src/main/resources/logback.xml @@ -967,6 +967,9 @@ + + +