From c896b81ddc596e4260dc67da0a02da5995e09d46 Mon Sep 17 00:00:00 2001 From: Joshua Karp Date: Thu, 11 Nov 2021 14:53:03 +1100 Subject: [PATCH] Supporting seed nodes: 1. Adding support for hostnames 2. Adding parsers for seed node sources 3. --seed-nodes option added to pk agent start 4. Separating NodeGraph db sync from NodeGraph.start 5. Adding Fixing timeouts for connections. 6. Separating seed node connection establishment from NodeManager.start, and adding to PolykeyAgent.start instead (dependency on ForwardProxy being started) 7. Re-adding the seed nodes into config.ts and setting PK_SEED_NODES to be empty by default (for CLI testing to connect to 0 seed nodes by default) --- .gitignore | 1 + src/PolykeyAgent.ts | 6 + src/agent/agentService.ts | 2 +- src/bin/agent/CommandStart.ts | 12 + src/bin/utils/options.ts | 30 ++- src/bin/utils/parsers.ts | 99 ++++++- src/client/clientService.ts | 13 - src/client/rpcNodes.ts | 26 +- src/config.ts | 15 ++ src/errors.ts | 3 + src/network/errors.ts | 3 + src/network/types.ts | 4 + src/network/utils.ts | 48 +++- src/nodes/NodeConnection.ts | 47 ++-- src/nodes/NodeGraph.ts | 71 +++-- src/nodes/NodeManager.ts | 133 +++++----- src/nodes/errors.ts | 4 + src/nodes/types.ts | 9 +- src/nodes/utils.ts | 11 - tests/agent/GRPCClientAgent.test.ts | 2 +- tests/bin/agent/start.test.ts | 244 ++++++++++++++++++ tests/bin/nodes.test.ts | 4 +- tests/bin/notifications.test.ts | 2 +- tests/bin/utils.ts | 4 + tests/bin/vaults.test.ts | 6 +- tests/client/rpcNodes.test.ts | 18 +- tests/client/rpcNotifications.test.ts | 6 +- tests/network/utils.test.ts | 12 +- tests/nodes/NodeConnection.test.ts | 17 +- tests/nodes/NodeGraph.test.ts | 160 ++++++++---- tests/nodes/NodeManager.test.ts | 24 +- .../NotificationsManager.test.ts | 2 +- tests/utils.ts | 2 +- tests/vaults/VaultManager.test.ts | 2 +- 34 files changed, 807 insertions(+), 235 deletions(-) diff --git a/.gitignore b/.gitignore index 18adc64cf..cf000140c 100644 --- a/.gitignore +++ b/.gitignore @@ -123,3 +123,4 @@ dist # editor .vscode/ +.idea/ diff --git a/src/PolykeyAgent.ts b/src/PolykeyAgent.ts index 5dff3aa9c..d32229ff6 100644 --- a/src/PolykeyAgent.ts +++ b/src/PolykeyAgent.ts @@ -1,6 +1,7 @@ import type { FileSystem } from './types'; import type { PolykeyWorkerManagerInterface } from './workers/types'; import type { Host, Port } from './network/types'; +import type { NodeMapping } from './nodes/types'; import path from 'path'; import process from 'process'; @@ -58,6 +59,7 @@ class PolykeyAgent { networkConfig = {}, forwardProxyConfig = {}, reverseProxyConfig = {}, + seedNodes = {}, // Optional dependencies status, schema, @@ -99,6 +101,7 @@ class PolykeyAgent { connTimeoutTime?: number; }; networkConfig?: NetworkConfig; + seedNodes?: NodeMapping; status?: Status; schema?: Schema; keyManager?: KeyManager; @@ -241,6 +244,7 @@ class PolykeyAgent { nodeManager ?? (await NodeManager.createNodeManager({ db, + seedNodes, sigchain, keyManager, fwdProxy, @@ -503,6 +507,8 @@ class PolykeyAgent { }); await this.nodeManager.start({ fresh }); + await this.nodeManager.getConnectionsToSeedNodes(); + await this.nodeManager.syncNodeGraph(); await this.vaultManager.start({ fresh }); await this.notificationsManager.start({ fresh }); await this.sessionManager.start({ fresh }); diff --git a/src/agent/agentService.ts b/src/agent/agentService.ts index 1683f3bb3..97811084f 100644 --- a/src/agent/agentService.ts +++ b/src/agent/agentService.ts @@ -195,7 +195,7 @@ function createAgentService({ ); for (const node of closestNodes) { const addressMessage = new nodesPB.Address(); - addressMessage.setHost(node.address.ip); + addressMessage.setHost(node.address.host); addressMessage.setPort(node.address.port); // Add the node to the response's map (mapping of node ID -> node address) response.getNodeTableMap().set(node.id, addressMessage); diff --git a/src/bin/agent/CommandStart.ts b/src/bin/agent/CommandStart.ts index cf4f54e07..df69d403c 100644 --- a/src/bin/agent/CommandStart.ts +++ b/src/bin/agent/CommandStart.ts @@ -23,6 +23,9 @@ class CommandStart extends CommandPolykey { this.addOption(binOptions.clientPort); this.addOption(binOptions.ingressHost); this.addOption(binOptions.ingressPort); + this.addOption(binOptions.connTimeoutTime); + this.addOption(binOptions.seedNodes); + this.addOption(binOptions.network); this.addOption(binOptions.background); this.addOption(binOptions.backgroundOutFile); this.addOption(binOptions.backgroundErrFile); @@ -66,6 +69,8 @@ class CommandStart extends CommandPolykey { options.recoveryCodeFile, this.fs, ); + const [seedNodes, defaults] = options.seedNodes; + if (defaults) Object.assign(seedNodes, options.network); const agentConfig = { password, nodePath: options.nodePath, @@ -73,12 +78,19 @@ class CommandStart extends CommandPolykey { rootKeyPairBits: options.rootKeyPairBits, recoveryCode: recoveryCodeIn, }, + forwardProxyConfig: { + connTimeoutTime: options.connTimeoutTime, + }, + reverseProxyConfig: { + connTimeoutTime: options.connTimeoutTime, + }, networkConfig: { clientHost: options.clientHost, clientPort: options.clientPort, ingressHost: options.ingressHost, ingressPort: options.ingressPort, }, + seedNodes, fresh: options.fresh, }; let recoveryCodeOut: RecoveryCode | undefined; diff --git a/src/bin/utils/options.ts b/src/bin/utils/options.ts index 2dc3fed43..ea95d2d9e 100644 --- a/src/bin/utils/options.ts +++ b/src/bin/utils/options.ts @@ -4,7 +4,7 @@ * @module */ import commander from 'commander'; -import * as binParsers from './parsers'; +import * as binParsers from '../utils/parsers'; import config from '../../config'; /** @@ -83,6 +83,13 @@ const ingressPort = new commander.Option( .env('PK_INGRESS_PORT') .default(config.defaults.networkConfig.ingressPort); +const connTimeoutTime = new commander.Option( + '--connection-timeout ', + 'Timeout value for connection establishment between nodes', +) + .argParser(binParsers.parseNumber) + .default(config.defaults.forwardProxyConfig.connTimeoutTime); + const passwordFile = new commander.Option( '-pf, --password-file ', 'Path to Password', @@ -118,6 +125,22 @@ const rootKeyPairBits = new commander.Option( 'Bit size of root key pair', ).argParser(binParsers.parseNumber); +const seedNodes = new commander.Option( + '-sn, --seed-nodes [nodeId1@host:port;nodeId2@host:port;...]', + 'Seed node address mappings', +) + .argParser(binParsers.parseSeedNodes) + .env('PK_SEED_NODES') + .default([{}, true]); + +const network = new commander.Option( + '-n --network ', + 'Setting the desired default network.', +) + .argParser(binParsers.parseNetwork) + .env('PK_NETWORK') + .default(config.defaults.network.mainnet); + export { nodePath, format, @@ -128,11 +151,14 @@ export { clientPort, ingressHost, ingressPort, + connTimeoutTime, + recoveryCodeFile, passwordFile, passwordNewFile, - recoveryCodeFile, background, backgroundOutFile, backgroundErrFile, rootKeyPairBits, + seedNodes, + network, }; diff --git a/src/bin/utils/parsers.ts b/src/bin/utils/parsers.ts index 7122042c2..3778d8000 100644 --- a/src/bin/utils/parsers.ts +++ b/src/bin/utils/parsers.ts @@ -1,6 +1,11 @@ import type { IdentityId, ProviderId } from '../../identities/types'; +import type { Host, Hostname, Port } from '../../network/types'; +import type { NodeAddress, NodeId, NodeMapping } from '../../nodes/types'; import commander from 'commander'; import * as nodesUtils from '../../nodes/utils'; +import * as networkUtils from '../../network/utils'; +import config from '../../config'; +import { never } from '../../utils'; function parseNumber(v: string): number { const num = parseInt(v); @@ -56,4 +61,96 @@ function parseIdentityString(identityString: string): { return { providerId, identityId }; } -export { parseNumber, parseSecretPath, parseGestaltId }; +/** + * Acquires the default seed nodes from src/config.ts. + */ +function getDefaultSeedNodes(network: string): NodeMapping { + const seedNodes: NodeMapping = {}; + let source; + switch (network) { + case 'testnet': + source = config.defaults.network.testnet; + break; + case 'mainnet': + source = config.defaults.network.mainnet; + break; + default: + never(); + } + for (const id in source) { + const seedNodeId = id as NodeId; + const seedNodeAddress: NodeAddress = { + host: source[seedNodeId].host as Host | Hostname, + port: source[seedNodeId].port as Port, + }; + seedNodes[seedNodeId] = seedNodeAddress; + } + return seedNodes; +} + +/** + * Seed nodes expected to be of form 'nodeId1@host:port;nodeId2@host:port;...' + * By default, any specified seed nodes (in CLI option, or environment variable) + * will overwrite the default nodes in src/config.ts. + * Special flag '' in the content indicates that the default seed + * nodes should be added to the starting seed nodes instead of being overwritten. + */ +function parseSeedNodes(rawSeedNodes: string): [NodeMapping, boolean] { + const seedNodeMappings: NodeMapping = {}; + let defaults = false; + // If specifically set no seed nodes, then ensure we start with none + if (rawSeedNodes === '') return [seedNodeMappings, defaults]; + const semicolonSeedNodes = rawSeedNodes.split(';'); + for (const rawSeedNode of semicolonSeedNodes) { + // Empty string will occur if there's an extraneous ';' (e.g. at end of env) + if (rawSeedNode === '') continue; + // Append the default seed nodes if we encounter the special flag + if (rawSeedNode === '') { + defaults = true; + continue; + } + const idHostPort = rawSeedNode.split(/[@:]/); + if (idHostPort.length !== 3) { + throw new commander.InvalidOptionArgumentError( + `${rawSeedNode} is not of format 'nodeId@host:port'`, + ); + } + if (!nodesUtils.isNodeId(idHostPort[0])) { + throw new commander.InvalidOptionArgumentError( + `${idHostPort[0]} is not a valid node ID`, + ); + } + if (!networkUtils.isValidHostname(idHostPort[1])) { + throw new commander.InvalidOptionArgumentError( + `${idHostPort[1]} is not a valid hostname`, + ); + } + const port = parseNumber(idHostPort[2]); + const seedNodeId = idHostPort[0] as NodeId; + const seedNodeAddress: NodeAddress = { + host: idHostPort[1] as Host | Hostname, + port: port as Port, + }; + seedNodeMappings[seedNodeId] = seedNodeAddress; + } + return [seedNodeMappings, defaults]; +} + +function parseNetwork(network: string): NodeMapping { + // Getting a list of network names from the config defaults + const networks = config.defaults.network; + const validNetworks = Object.keys(networks); + + // Checking if the network name is valid. + if (validNetworks.includes(network)) return getDefaultSeedNodes(network); + throw new commander.InvalidArgumentError(`${network} is not a valid network`); +} + +export { + parseNumber, + parseSecretPath, + parseGestaltId, + getDefaultSeedNodes, + parseSeedNodes, + parseNetwork, +}; diff --git a/src/client/clientService.ts b/src/client/clientService.ts index 1d776d09e..db7d1a549 100644 --- a/src/client/clientService.ts +++ b/src/client/clientService.ts @@ -14,7 +14,6 @@ import type { FileSystem } from '../types'; import type * as grpc from '@grpc/grpc-js'; import type { IClientServiceServer } from '../proto/js/polykey/v1/client_service_grpc_pb'; -import { promisify } from 'util'; import createStatusRPC from './rpcStatus'; import createSessionsRPC from './rpcSessions'; import createVaultRPC from './rpcVaults'; @@ -25,7 +24,6 @@ import createIdentitiesRPC from './rpcIdentities'; import createNotificationsRPC from './rpcNotifications'; import * as clientUtils from './utils'; import * as grpcUtils from '../grpc/utils'; -import * as nodesPB from '../proto/js/polykey/v1/nodes/nodes_pb'; import * as utilsPB from '../proto/js/polykey/v1/utils/utils_pb'; import { ClientServiceService } from '../proto/js/polykey/v1/client_service_grpc_pb'; @@ -114,17 +112,6 @@ function createClientService({ notificationsManager, authenticate, }), - nodesList: async ( - call: grpc.ServerWritableStream, - ): Promise => { - // Call.request // PROCESS THE REQEUST MESSAGE - const nodeMessage = new nodesPB.Node(); - nodeMessage.setNodeId('some node name'); - const write = promisify(call.write).bind(call); - await write(nodeMessage); - call.end(); - return; - }, agentStop: async ( call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData, diff --git a/src/client/rpcNodes.ts b/src/client/rpcNodes.ts index 6579a59d1..34458e0ab 100644 --- a/src/client/rpcNodes.ts +++ b/src/client/rpcNodes.ts @@ -7,10 +7,9 @@ import type * as grpc from '@grpc/grpc-js'; import type * as utils from '../client/utils'; import * as utilsPB from '../proto/js/polykey/v1/utils/utils_pb'; import * as nodesPB from '../proto/js/polykey/v1/nodes/nodes_pb'; -import * as nodesUtils from '../nodes/utils'; +import { utils as nodesUtils, errors as nodesErrors } from '../nodes'; import * as grpcUtils from '../grpc/utils'; -import * as nodesErrors from '../nodes/errors'; -import { makeNodeId } from '../nodes/utils'; +import * as networkUtils from '../network/utils'; const createNodesRPC = ({ nodeManager, @@ -40,16 +39,19 @@ const createNodesRPC = ({ if (!validNodeId) { throw new nodesErrors.ErrorInvalidNodeId(); } - const validHost = nodesUtils.isValidHost( + const validHost = networkUtils.isValidHost( call.request.getAddress()!.getHost(), ); if (!validHost) { throw new nodesErrors.ErrorInvalidHost(); } - await nodeManager.setNode(makeNodeId(call.request.getNodeId()), { - ip: call.request.getAddress()!.getHost(), - port: call.request.getAddress()!.getPort(), - } as NodeAddress); + await nodeManager.setNode( + nodesUtils.makeNodeId(call.request.getNodeId()), + { + host: call.request.getAddress()!.getHost(), + port: call.request.getAddress()!.getPort(), + } as NodeAddress, + ); callback(null, response); return; } catch (err) { @@ -70,7 +72,7 @@ const createNodesRPC = ({ call.sendMetadata(metadata); const status = await nodeManager.pingNode( - makeNodeId(call.request.getNodeId()), + nodesUtils.makeNodeId(call.request.getNodeId()), ); response.setSuccess(status); callback(null, response); @@ -94,7 +96,7 @@ const createNodesRPC = ({ const metadata = await authenticate(call.metadata); call.sendMetadata(metadata); - const remoteNodeId = makeNodeId(call.request.getNodeId()); + const remoteNodeId = nodesUtils.makeNodeId(call.request.getNodeId()); const gestaltInvite = await notificationsManager.findGestaltInvite( remoteNodeId, ); @@ -133,12 +135,12 @@ const createNodesRPC = ({ const metadata = await authenticate(call.metadata); call.sendMetadata(metadata); - const nodeId = makeNodeId(call.request.getNodeId()); + const nodeId = nodesUtils.makeNodeId(call.request.getNodeId()); const address = await nodeManager.findNode(nodeId); response .setNodeId(nodeId) .setAddress( - new nodesPB.Address().setHost(address.ip).setPort(address.port), + new nodesPB.Address().setHost(address.host).setPort(address.port), ); callback(null, response); return; diff --git a/src/config.ts b/src/config.ts index db95f07e9..3f07c0ab7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -91,6 +91,21 @@ const config = { connConnectTime: 20000, connTimeoutTime: 20000, }, + // Note: this is not used by the `PolykeyAgent`, that is defaulting to `{}`. + network: { + mainnet: { + v359vgrgmqf1r5g4fvisiddjknjko6bmm4qv7646jr7fi9enbfuug: { + host: 'testnet.polykey.io', + port: 1314, + }, + }, + testnet: { + v359vgrgmqf1r5g4fvisiddjknjko6bmm4qv7646jr7fi9enbfuug: { + host: '127.0.0.3', + port: 1314, + }, + }, + }, }, }; diff --git a/src/errors.ts b/src/errors.ts index 2096d10c9..d29fe973a 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -20,6 +20,8 @@ class ErrorPolykeyClientDestroyed extends ErrorPolykey {} class ErrorInvalidId extends ErrorPolykey {} +class ErrorInvalidConfigEnvironment extends ErrorPolykey {} + export { sysexits, ErrorPolykey, @@ -31,6 +33,7 @@ export { ErrorPolykeyClientNotRunning, ErrorPolykeyClientDestroyed, ErrorInvalidId, + ErrorInvalidConfigEnvironment, }; /** diff --git a/src/network/errors.ts b/src/network/errors.ts index 219e6c8b8..9b03420b4 100644 --- a/src/network/errors.ts +++ b/src/network/errors.ts @@ -83,6 +83,8 @@ class ErrorCertChainKeyInvalid extends ErrorCertChain {} */ class ErrorCertChainSignatureInvalid extends ErrorCertChain {} +class ErrorHostnameResolutionFailed extends ErrorNetwork {} + export { ErrorNetwork, ErrorForwardProxyNotStarted, @@ -110,4 +112,5 @@ export { ErrorCertChainNameInvalid, ErrorCertChainKeyInvalid, ErrorCertChainSignatureInvalid, + ErrorHostnameResolutionFailed, }; diff --git a/src/network/types.ts b/src/network/types.ts index c163a62b2..c4273b539 100644 --- a/src/network/types.ts +++ b/src/network/types.ts @@ -6,7 +6,10 @@ import type { } from '../keys/types'; import type { Opaque } from '../types'; +// Host is always an IP address type Host = Opaque<'Host', string>; +// Specifically for hostname domain names (i.e. to be resolved to an IP address) +type Hostname = Opaque<'Hostname', string>; type Port = Opaque<'Port', number>; type Address = Opaque<'Address', string>; @@ -42,6 +45,7 @@ type NetworkMessage = PingMessage | PongMessage; export type { Host, + Hostname, Port, Address, TLSConfig, diff --git a/src/network/utils.ts b/src/network/utils.ts index f5d20c63d..baba7f03c 100644 --- a/src/network/utils.ts +++ b/src/network/utils.ts @@ -1,13 +1,14 @@ import type { Socket } from 'net'; import type { TLSSocket } from 'tls'; -import type { Host, Port, Address, NetworkMessage } from './types'; +import type { Host, Hostname, Port, Address, NetworkMessage } from './types'; import type { Certificate, PublicKey } from '../keys/types'; import type { NodeId } from '../nodes/types'; import { Buffer } from 'buffer'; +import dns from 'dns'; import { IPv4, IPv6, Validator } from 'ip-num'; import * as networkErrors from './errors'; -import { isEmptyObject } from '../utils'; +import { isEmptyObject, promisify } from '../utils'; import { utils as keysUtils } from '../keys'; const pingBuffer = serializeNetworkMessage({ @@ -53,6 +54,46 @@ function parseAddress(address: string): [Host, Port] { return [dstHost as Host, dstPort as Port]; } +/** + * Validates that a provided host address is a valid IPv4 or IPv6 address. + */ +function isValidHost(host: string): boolean { + const [isIPv4] = Validator.isValidIPv4String(host); + const [isIPv6] = Validator.isValidIPv6String(host); + return isIPv4 || isIPv6; +} + +/** + * Validates that a provided hostname is valid, as per RFC 1123. + */ +function isValidHostname(hostname: string): boolean { + return hostname.match( + /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/, + ) + ? true + : false; +} + +/** + * Resolves a provided hostname to its respective IP address (type Host). + */ +async function resolveHost(host: Host | Hostname): Promise { + // If already IPv4/IPv6 address, return it + if (isValidHost(host)) { + return host as Host; + } + const lookup = promisify(dns.lookup).bind(dns); + let resolvedHost; + try { + // Resolve the hostname and get the IPv4 address + resolvedHost = await lookup(host, 4); + } catch (e) { + throw new networkErrors.ErrorHostnameResolutionFailed(e.message); + } + // Returns an array of [ resolved address, family (4 or 6) ] + return resolvedHost[0] as Host; +} + /** * Zero IPs should be resolved to localhost when used as the target * This is usually done automatically, but utp-native doesn't do this @@ -317,6 +358,9 @@ export { toAuthToken, buildAddress, parseAddress, + isValidHost, + isValidHostname, + resolveHost, resolvesZeroIP, serializeNetworkMessage, unserializeNetworkMessage, diff --git a/src/nodes/NodeConnection.ts b/src/nodes/NodeConnection.ts index 30f10b442..d4c9b61b8 100644 --- a/src/nodes/NodeConnection.ts +++ b/src/nodes/NodeConnection.ts @@ -1,5 +1,5 @@ import type { NodeId, NodeData } from './types'; -import type { Host, Port, ProxyConfig } from '../network/types'; +import type { Host, Hostname, Port, ProxyConfig } from '../network/types'; import type { KeyManager } from '../keys'; import type { SignedNotification } from '../notifications/types'; import type { ChainDataEncoded } from '../sigchain/types'; @@ -42,8 +42,11 @@ class NodeConnection { protected keyManager: KeyManager; // Node ID, host, and port of the target node at the end of this connection + // Hostname defined if the target's host was resolved from this hostname. + // Undefined if an IP address was initially provided protected targetNodeId: NodeId; protected ingressHost: Host; + protected ingressHostname: Hostname | undefined; protected ingressPort: Port; // Host and port of the initiating node (client) where the connection begins @@ -57,36 +60,41 @@ class NodeConnection { static async createNodeConnection({ targetNodeId, targetHost, + targetHostname = undefined, targetPort, + connTimeout = 20000, forwardProxy, keyManager, logger = new Logger(this.name), - brokerConnections = new Map(), + seedConnections = new Map(), }: { targetNodeId: NodeId; targetHost: Host; + targetHostname?: Hostname; targetPort: Port; + connTimeout?: number; forwardProxy: ForwardProxy; keyManager: KeyManager; logger?: Logger; - brokerConnections?: Map; + seedConnections?: Map; }): Promise { logger.info(`Creating ${this.name}`); - const proxyConfig = { + const proxyConfig: ProxyConfig = { host: forwardProxy.proxyHost, port: forwardProxy.proxyPort, authToken: forwardProxy.authToken, - } as ProxyConfig; + }; const nodeConnection = new NodeConnection({ + targetNodeId, + targetHost, + targetHostname, + targetPort, forwardProxy, keyManager, logger, - targetHost, - targetNodeId, - targetPort, proxyConfig, }); - await nodeConnection.start({ brokerConnections }); + await nodeConnection.start({ seedConnections, connTimeout }); logger.info(`Created ${this.name}`); return nodeConnection; } @@ -94,6 +102,7 @@ class NodeConnection { constructor({ targetNodeId, targetHost, + targetHostname = undefined, targetPort, forwardProxy, keyManager, @@ -102,6 +111,7 @@ class NodeConnection { }: { targetNodeId: NodeId; targetHost: Host; + targetHostname?: Hostname; targetPort: Port; forwardProxy: ForwardProxy; keyManager: KeyManager; @@ -111,6 +121,7 @@ class NodeConnection { this.logger = logger; this.targetNodeId = targetNodeId; this.ingressHost = targetHost; + this.ingressHostname = targetHostname; this.ingressPort = targetPort; this.fwdProxy = forwardProxy; this.keyManager = keyManager; @@ -120,17 +131,19 @@ class NodeConnection { /** * Initialises and starts the connection (via the fwdProxy). * - * @param brokerConnections map of all established broker connections + * @param seedConnections map of all established seed node connections * If not provided, it's assumed a direct connection can be made to the target - * (i.e. without hole punching), as the broker nodes relay the hole punch message. + * (i.e. without hole punching, and therefore not being a NAT), as the seed + * nodes relay the hole punch message. */ public async start({ - brokerConnections = new Map(), + seedConnections = new Map(), + connTimeout, }: { - brokerConnections?: Map; + seedConnections?: Map; + connTimeout?: number; } = {}) { this.logger.info(`Starting ${this.constructor.name}`); - // 1. Get the egress port of the fwdProxy (used for hole punching) const egressAddress = networkUtils.buildAddress( this.fwdProxy.egressHost, @@ -154,9 +167,9 @@ class NodeConnection { port: this.ingressPort, proxyConfig: this.proxyConfig, logger: this.logger.getChild(GRPCClientAgent.name), - timeout: 20000, + timeout: connTimeout, }), - Array.from(brokerConnections, ([_, conn]) => + Array.from(seedConnections, ([_, conn]) => conn.sendHolePunchMessage( this.keyManager.getNodeId(), this.targetNodeId, @@ -262,7 +275,7 @@ class NodeConnection { nodes.push({ id: nodeId as NodeId, address: { - ip: address.getHost() as Host, + host: address.getHost() as Host | Hostname, port: address.getPort() as Port, }, distance: nodesUtils.calculateDistance(targetNodeId, nodeId as NodeId), diff --git a/src/nodes/NodeGraph.ts b/src/nodes/NodeGraph.ts index 932a75d8b..1784bf28d 100644 --- a/src/nodes/NodeGraph.ts +++ b/src/nodes/NodeGraph.ts @@ -1,5 +1,5 @@ import type { NodeId, NodeAddress, NodeBucket, NodeData } from './types'; -import type { Host, Port } from '../network/types'; +import type { Host, Hostname, Port } from '../network/types'; import type { DB, DBLevel, DBOp } from '@matrixai/db'; import type { NodeConnection } from '../nodes'; @@ -42,7 +42,6 @@ class NodeGraph { protected nodeGraphDb: DBLevel; protected nodeGraphBucketsDb: DBLevel; protected lock: Mutex = new Mutex(); - protected _started: boolean = false; public static async createNodeGraph({ db, @@ -101,17 +100,6 @@ class NodeGraph { } this.nodeGraphDb = nodeGraphDb; this.nodeGraphBucketsDb = nodeGraphBucketsDb; - // TODO: change these to seed nodes - // populate this node's bucket database - // gets the k closest nodes from each broker - // and adds each of them to the database - for (const [, conn] of this.nodeManager.getBrokerNodeConnections()) { - const nodes = await conn.getClosestNodes(this.nodeManager.getNodeId()); - for (const n of nodes) { - await this.setNode(n.id, n.address); - await this.nodeManager.getConnectionToNode(n.id); - } - } this.logger.info(`Started ${this.constructor.name}`); } @@ -155,6 +143,36 @@ class NodeGraph { } } + /** + * Perform an initial database synchronisation: get the k closest nodes + * from each seed node and add them to this database + * For now, we also attempt to establish a connection to each of them. + * If these nodes are offline, this will impose a performance penalty, + * so we should investigate performing this in the background if possible. + * Alternatively, we can also just add the nodes to our database without + * establishing connection. + * This has been removed from start() as there's a chicken-egg scenario + * where we require the NodeGraph instance to be created in order to get + * connections. + */ + public async syncNodeGraph() { + for (const [, conn] of await this.nodeManager.getConnectionsToSeedNodes()) { + const nodes = await conn.getClosestNodes(this.nodeManager.getNodeId()); + for (const n of nodes) { + await this.setNode(n.id, n.address); + try { + await this.nodeManager.getConnectionToNode(n.id); + } catch (e) { + if (e instanceof nodesErrors.ErrorNodeConnectionTimeout) { + continue; + } else { + throw e; + } + } + } + } + } + @ready(new nodesErrors.ErrorNodeGraphNotRunning()) public getNodeId(): NodeId { return this.nodeManager.getNodeId(); @@ -184,7 +202,9 @@ class NodeGraph { ); // Cast the non-primitive types correctly (ensures type safety when using them) for (const nodeId in bucket) { - bucket[nodeId].address.ip = bucket[nodeId].address.ip as Host; + bucket[nodeId].address.host = bucket[nodeId].address.host as + | Host + | Hostname; bucket[nodeId].address.port = bucket[nodeId].address.port as Port; bucket[nodeId].lastUpdated = new Date(bucket[nodeId].lastUpdated); } @@ -219,17 +239,26 @@ class NodeGraph { if (bucket == null) { bucket = {}; } - const bucketEntries = Object.entries(bucket); - if (bucketEntries.length === this.maxNodesPerBucket) { - const leastActive = bucketEntries.reduce((prev, curr) => { - return prev[1].lastUpdated < curr[1].lastUpdated ? prev : curr; - }); - delete bucket[leastActive[0]]; - } bucket[nodeId] = { address: nodeAddress, lastUpdated: new Date(), }; + // Perform the check on size after we add/update the node. If it's an update, + // then we don't need to perform the deletion. + let bucketEntries = Object.entries(bucket); + if (bucketEntries.length > this.maxNodesPerBucket) { + const leastActive = bucketEntries.reduce((prev, curr) => { + return new Date(prev[1].lastUpdated) < new Date(curr[1].lastUpdated) + ? prev + : curr; + }); + delete bucket[leastActive[0]]; + bucketEntries = Object.entries(bucket); + // For safety, make sure that the bucket is actually at maxNodesPerBucket + if (bucketEntries.length !== this.maxNodesPerBucket) { + throw new nodesErrors.ErrorNodeGraphOversizedBucket(); + } + } return [ { type: 'put', diff --git a/src/nodes/NodeManager.ts b/src/nodes/NodeManager.ts index e5161f64e..f00994697 100644 --- a/src/nodes/NodeManager.ts +++ b/src/nodes/NodeManager.ts @@ -6,12 +6,13 @@ import type { ClaimIdString } from '../claims/types'; import type { NodeId, NodeAddress, + NodeMapping, NodeData, NodeBucket, NodeConnectionMap, } from '../nodes/types'; import type { SignedNotification } from '../notifications/types'; -import type { Host, Port } from '../network/types'; +import type { Host, Hostname, Port } from '../network/types'; import type { Timer } from '../types'; import type { DB } from '@matrixai/db'; @@ -28,7 +29,7 @@ import { import NodeGraph from './NodeGraph'; import NodeConnection from './NodeConnection'; import * as nodesErrors from './errors'; -import * as networkErrors from '../network/errors'; +import { utils as networkUtils, errors as networkErrors } from '../network'; import * as sigchainUtils from '../sigchain/utils'; import * as claimsUtils from '../claims/utils'; @@ -38,9 +39,6 @@ interface NodeManager extends CreateDestroyStartStop {} new nodesErrors.ErrorNodeManagerDestroyed(), ) class NodeManager { - // LevelDB directory to store all the information for managing nodes - // public readonly nodesPath: string; - protected db: DB; protected logger: Logger; protected lock: Mutex = new Mutex(); @@ -52,40 +50,38 @@ class NodeManager { protected revProxy: ReverseProxy; // Active connections to other nodes - // protected connections: Map = new Map(); protected connections: NodeConnectionMap = new Map(); - // Node ID -> node address mappings for the bootstrap/broker nodes - protected brokerNodes: NodeBucket = {}; - protected brokerNodeConnections: Map = new Map(); + // Node ID -> node address mappings for the seed nodes + protected seedNodes: NodeMapping = {}; static async createNodeManager({ db, + seedNodes = {}, keyManager, sigchain, fwdProxy, revProxy, logger = new Logger(this.name), - brokerNodes = {}, fresh = false, }: { db: DB; + seedNodes?: NodeMapping; keyManager: KeyManager; sigchain: Sigchain; fwdProxy: ForwardProxy; revProxy: ReverseProxy; logger?: Logger; - brokerNodes?: NodeBucket; fresh?: boolean; }): Promise { logger.info(`Creating ${this.name}`); const nodeManager = new NodeManager({ db, + seedNodes, keyManager, sigchain, fwdProxy, revProxy, logger, - brokerNodes, }); await nodeManager.start({ fresh, @@ -96,28 +92,28 @@ class NodeManager { constructor({ db, + seedNodes, keyManager, sigchain, fwdProxy, revProxy, logger, - brokerNodes, }: { db: DB; + seedNodes: NodeMapping; keyManager: KeyManager; sigchain: Sigchain; fwdProxy: ForwardProxy; revProxy: ReverseProxy; logger: Logger; - brokerNodes: NodeBucket; }) { - this.logger = logger; this.db = db; + this.seedNodes = seedNodes; this.keyManager = keyManager; this.sigchain = sigchain; this.fwdProxy = fwdProxy; this.revProxy = revProxy; - this.brokerNodes = brokerNodes; + this.logger = logger; } get locked(): boolean { @@ -130,21 +126,18 @@ class NodeManager { fresh?: boolean; } = {}) { this.logger.info(`Starting ${this.constructor.name}`); - // Establish and start connections to the brokers - for (const brokerId in this.brokerNodes) { - await this.createConnectionToBroker( - brokerId as NodeId, - this.brokerNodes[brokerId].address, - ); - } - // Instantiate the node graph (containing Kademlia implementation) this.nodeGraph = await NodeGraph.createNodeGraph({ db: this.db, nodeManager: this, logger: this.logger, + fresh, }); - await this.nodeGraph.start({ fresh }); + // Add the seed nodes to the NodeGraph + for (const id in this.seedNodes) { + const seedNodeId = id as NodeId; + await this.nodeGraph.setNode(seedNodeId, this.seedNodes[seedNodeId]); + } this.logger.info(`Started ${this.constructor.name}`); } @@ -159,9 +152,6 @@ class NodeManager { // also still valid on restart though. this.connections.delete(targetNodeId); } - for (const [, conn] of this.brokerNodeConnections) { - await conn.stop(); - } await this.nodeGraph.stop(); this.logger.info(`Stopped ${this.constructor.name}`); } @@ -229,7 +219,7 @@ class NodeManager { // i.e. no NodeConnection object created (no need for GRPCClient) await this.fwdProxy.openConnection( targetNodeId, - targetAddress.ip, + await networkUtils.resolveHost(targetAddress.host), targetAddress.port, ); } catch (e) { @@ -470,13 +460,19 @@ class NodeManager { lock: MutexInterface, ): Promise { const targetAddress = await this.findNode(targetNodeId); + // If the stored host is not a valid host (IP address), then we assume it to + // be a hostname + const targetHostname = !(await networkUtils.isValidHost(targetAddress.host)) + ? (targetAddress.host as Hostname) + : undefined; const connection = await NodeConnection.createNodeConnection({ targetNodeId: targetNodeId, - targetHost: targetAddress.ip, + targetHost: await networkUtils.resolveHost(targetAddress.host), + targetHostname: targetHostname, targetPort: targetAddress.port, forwardProxy: this.fwdProxy, keyManager: this.keyManager, - brokerConnections: this.brokerNodeConnections, + seedConnections: await this.getConnectionsToSeedNodes(), logger: this.logger, }); // Add it to the map of active connections @@ -485,46 +481,51 @@ class NodeManager { } /** - * Create and start a connection to a broker node. Assumes that a direct - * connection to the broker can be established (i.e. no hole punching required). - * - * @param brokerNodeId ID of the broker node to connect to - * @param brokerNodeAddress host and port of the broker node to connect to - * @returns + * Acquires a map of connections to the seed nodes. + * These connections are expected to have already been established in start(), + * so this should simply be a constant-time retrieval from the NodeConnectionMap. */ @ready(new nodesErrors.ErrorNodeManagerNotRunning()) - public async createConnectionToBroker( - brokerNodeId: NodeId, - brokerNodeAddress: NodeAddress, - ): Promise { - return await this._transaction(async () => { - // Throw error if trying to connect to self - if (brokerNodeId === this.getNodeId()) { - throw new nodesErrors.ErrorNodeGraphSelfConnect(); - } - // Attempt to get an existing connection - const existingConnection = this.brokerNodeConnections.get(brokerNodeId); - if (existingConnection != null) { - return existingConnection; + public async getConnectionsToSeedNodes(): Promise< + Map + > { + const connections: Map = new Map(); + // GetConnectionToNode internally calls this function if the connection to + // some node does not already exist (i.e. there's no existing entry in the + // NodeConnectionMap). Therefore, we have the potential for a deadlock if a + // connection to a seed node has been lost or doesn't already exist and + // this function is called: there would be 2 nested calls to + // getConnectionToNode on the seed node, causing a deadlock. To prevent this, + // we do a fail-safe here, where we temporarily clear this.seedNodes, such + // that we don't attempt to use the seed nodes to connect to another seed node. + const seedNodesCopy = this.seedNodes; + this.seedNodes = {}; + try { + for (const id in this.seedNodes) { + const seedNodeId = id as NodeId; + try { + connections.set( + seedNodeId, + await this.getConnectionToNode(seedNodeId), + ); + } catch (e) { + // If we can't connect to a seed node, simply skip it + if (e instanceof nodesErrors.ErrorNodeConnectionTimeout) { + continue; + } + throw e; + } } - const brokerConnection = await NodeConnection.createNodeConnection({ - targetNodeId: brokerNodeId, - targetHost: brokerNodeAddress.ip, - targetPort: brokerNodeAddress.port, - forwardProxy: this.fwdProxy, - keyManager: this.keyManager, - logger: this.logger, - }); - // TODO: may need to change this start() to some kind of special 'direct - // connection' mechanism (currently just does the same openConnection() call - // as any other node, but without hole punching). - this.brokerNodeConnections.set(brokerNodeId, brokerConnection); - return brokerConnection; - }); + } finally { + // Even if an exception is thrown, ensure the seed node mappings are reinstated + this.seedNodes = seedNodesCopy; + } + return connections; } - public getBrokerNodeConnections(): Map { - return this.brokerNodeConnections; + @ready(new nodesErrors.ErrorNodeManagerNotRunning()) + public async syncNodeGraph() { + await this.nodeGraph.syncNodeGraph(); } /** diff --git a/src/nodes/errors.ts b/src/nodes/errors.ts index be4e81b59..04bf0db25 100644 --- a/src/nodes/errors.ts +++ b/src/nodes/errors.ts @@ -29,6 +29,9 @@ class ErrorNodeGraphInvalidBucketIndex extends ErrorNodes {} class ErrorNodeConnectionRunning extends ErrorNodes {} class ErrorNodeConnectionNotRunning extends ErrorNodes {} +class ErrorNodeGraphOversizedBucket extends ErrorNodes { + description: 'Bucket invalidly contains more nodes than capacity'; +} class ErrorNodeConnectionDestroyed extends ErrorNodes {} @@ -67,6 +70,7 @@ export { ErrorNodeGraphInvalidBucketIndex, ErrorNodeConnectionRunning, ErrorNodeConnectionNotRunning, + ErrorNodeGraphOversizedBucket, ErrorNodeConnectionDestroyed, ErrorNodeConnectionTimeout, ErrorNodeConnectionNotExist, diff --git a/src/nodes/types.ts b/src/nodes/types.ts index de73d6677..a2c5654a0 100644 --- a/src/nodes/types.ts +++ b/src/nodes/types.ts @@ -1,7 +1,7 @@ import type NodeConnection from './NodeConnection'; import type { MutexInterface } from 'async-mutex'; import type { Opaque } from '../types'; -import type { Host, Port } from '../network/types'; +import type { Host, Hostname, Port } from '../network/types'; import type { Claim, ClaimId } from '../claims/types'; import type { ChainData } from '../sigchain/types'; import type { IdString } from '../GenericIdTypes'; @@ -9,10 +9,14 @@ import type { IdString } from '../GenericIdTypes'; type NodeId = Opaque<'NodeId', IdString>; type NodeAddress = { - ip: Host; + host: Host | Hostname; port: Port; }; +type NodeMapping = { + [key: string]: NodeAddress; +}; + type NodeData = { id: NodeId; address: NodeAddress; @@ -83,6 +87,7 @@ type NodeGraphOp = export type { NodeId, NodeAddress, + NodeMapping, NodeData, NodeClaim, NodeInfo, diff --git a/src/nodes/utils.ts b/src/nodes/utils.ts index 35b588623..b0a5e9b98 100644 --- a/src/nodes/utils.ts +++ b/src/nodes/utils.ts @@ -1,6 +1,5 @@ import type { NodeData, NodeId } from './types'; -import { Validator } from 'ip-num'; import { ErrorInvalidNodeId } from './errors'; import { fromMultibase, isIdString, makeIdString } from '../GenericIdTypes'; @@ -66,15 +65,6 @@ function makeNodeId(arg: any): NodeId { return makeIdString(arg, 32, 'base32hex'); } -/** - * Validates that a provided host address is a valid IPv4 or IPv6 address. - */ -function isValidHost(host: string): boolean { - const [isIPv4] = Validator.isValidIPv4String(host); - const [isIPv6] = Validator.isValidIPv6String(host); - return isIPv4 || isIPv6; -} - /** * Node ID to an array of 8-bit unsigned ints */ @@ -103,7 +93,6 @@ export { calculateBucketIndex, isNodeId, makeNodeId, - isValidHost, nodeIdToU8, sortByDistance, }; diff --git a/tests/agent/GRPCClientAgent.test.ts b/tests/agent/GRPCClientAgent.test.ts index 264400932..22295ec4d 100644 --- a/tests/agent/GRPCClientAgent.test.ts +++ b/tests/agent/GRPCClientAgent.test.ts @@ -290,7 +290,7 @@ describe('GRPC agent', () => { lock: new Mutex(), }); await nodeManager.setNode(nodeIdY, { - ip: 'unnecessary' as Host, + host: 'unnecessary' as Host, port: 0 as Port, } as NodeAddress); }); diff --git a/tests/bin/agent/start.test.ts b/tests/bin/agent/start.test.ts index 8a842282a..9a4ffd272 100644 --- a/tests/bin/agent/start.test.ts +++ b/tests/bin/agent/start.test.ts @@ -6,6 +6,7 @@ import readline from 'readline'; import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; import { Status, errors as statusErrors } from '@/status'; import config from '@/config'; +import * as nodesUtils from '@/nodes/utils'; import * as testBinUtils from '../utils'; describe('start', () => { @@ -569,4 +570,247 @@ describe('start', () => { }, global.defaultTimeout * 2, ); + describe('seed nodes', () => { + let seedNodeClose; + const connTimeoutTime = 500; + let seedNodeId; + const seedNodeHost = '127.0.0.1'; + let seedNodePort; + + const dummySeed1Id = nodesUtils.makeNodeId( + 'vrsc24a1er424epq77dtoveo93meij0pc8ig4uvs9jbeld78n9nl0', + ); + const dummySeed1Host = '128.0.0.1'; + const dummySeed1Port = 1314; + const dummySeed2Id = nodesUtils.makeNodeId( + 'vrcacp9vsb4ht25hds6s4lpp2abfaso0mptcfnh499n35vfcn2gkg', + ); + const dummySeed2Host = '128.0.0.1'; + const dummySeed2Port = 1314; + + beforeAll(async () => { + seedNodeClose = await testBinUtils.pkAgent([ + '--connection-timeout', + connTimeoutTime.toString(), + '--ingress-host', + seedNodeHost, + ]); + const status = new Status({ + statusPath: path.join(global.binAgentDir, config.defaults.statusBase), + fs, + logger, + }); + const statusInfo = await status.waitFor('LIVE', 5000); + // Get the dynamic seed node components + seedNodeId = statusInfo.data.nodeId; + seedNodePort = statusInfo.data.ingressPort; + }, global.maxTimeout); + afterAll(async () => { + await seedNodeClose(); + }); + + test( + 'start with seed nodes as argument', + async () => { + const password = 'abc123'; + const passwordPath = path.join(dataDir, 'password'); + await fs.promises.writeFile(passwordPath, password); + const nodePath = path.join(dataDir, 'polykey'); + + await testBinUtils.pkStdio( + [ + 'agent', + 'start', + '--node-path', + nodePath, + '--password-file', + passwordPath, + '--root-key-pair-bits', + '1024', + '--seed-nodes', + `${seedNodeId}@${seedNodeHost}:${seedNodePort};${dummySeed1Id}@${dummySeed1Host}:${dummySeed1Port}`, + '--connection-timeout', + connTimeoutTime.toString(), + '--verbose', + ], + { + PK_SEED_NODES: `${dummySeed2Id}@${dummySeed2Host}:${dummySeed2Port}`, + }, + dataDir, + ); + const statusPath = path.join(nodePath, 'status.json'); + const status = new Status({ + statusPath, + fs, + logger, + }); + await status.waitFor('LIVE', 2000); + + // Check the seed nodes have been added to the node graph + const foundSeedNode = await testBinUtils.pkStdio([ + 'nodes', + 'find', + seedNodeId, + '--node-path', + nodePath, + '--password-file', + passwordPath, + '--verbose', + ]); + expect(foundSeedNode.exitCode).toBe(0); + expect(foundSeedNode.stdout).toContain( + `Found node at ${seedNodeHost}:${seedNodePort}`, + ); + const foundDummy1 = await testBinUtils.pkStdio([ + 'nodes', + 'find', + dummySeed1Id, + '--node-path', + nodePath, + '--password-file', + passwordPath, + '--verbose', + ]); + expect(foundDummy1.exitCode).toBe(0); + expect(foundDummy1.stdout).toContain( + `Found node at ${dummySeed1Host}:${dummySeed1Port}`, + ); + // Check the seed node in the environment variable was superseded by the + // ones provided as CLI arguments + const notFoundDummy2 = await testBinUtils.pkStdio([ + 'nodes', + 'find', + dummySeed2Id, + '--node-path', + nodePath, + '--password-file', + passwordPath, + '--verbose', + ]); + expect(notFoundDummy2.exitCode).toBe(1); + expect(notFoundDummy2.stdout).toContain( + `Failed to find node ${dummySeed2Id}`, + ); + await testBinUtils.pkStdio( + [ + 'agent', + 'stop', + '--node-path', + nodePath, + '--password-file', + passwordPath, + ], + undefined, + dataDir, + ); + await status.waitFor('DEAD', 5000); + }, + global.defaultTimeout * 2, + ); + + test( + 'start with seed nodes from environment variable and config file', + async () => { + const password = 'abc123'; + const passwordPath = path.join(dataDir, 'password'); + await fs.promises.writeFile(passwordPath, password); + const nodePath = path.join(dataDir, 'polykey'); + + await testBinUtils.pkStdio( + [ + 'agent', + 'start', + '--node-path', + nodePath, + '--password-file', + passwordPath, + '--root-key-pair-bits', + '1024', + '--connection-timeout', + connTimeoutTime.toString(), + '--verbose', + ], + { + PK_SEED_NODES: + `${seedNodeId}@${seedNodeHost}:${seedNodePort};` + + `${dummySeed1Id}@${dummySeed1Host}:${dummySeed1Port};` + + ``, + }, + dataDir, + ); + const statusPath = path.join(nodePath, 'status.json'); + const status = new Status({ + statusPath, + fs, + logger, + }); + await status.waitFor('LIVE', 2000); + + // Check the seed nodes have been added to the node graph + const foundSeedNode = await testBinUtils.pkStdio([ + 'nodes', + 'find', + seedNodeId, + '--node-path', + nodePath, + '--password-file', + passwordPath, + '--verbose', + ]); + expect(foundSeedNode.exitCode).toBe(0); + expect(foundSeedNode.stdout).toContain( + `Found node at ${seedNodeHost}:${seedNodePort}`, + ); + const foundDummy1 = await testBinUtils.pkStdio([ + 'nodes', + 'find', + dummySeed1Id, + '--node-path', + nodePath, + '--password-file', + passwordPath, + '--verbose', + ]); + expect(foundDummy1.exitCode).toBe(0); + expect(foundDummy1.stdout).toContain( + `Found node at ${dummySeed1Host}:${dummySeed1Port}`, + ); + // Check the seed node/s in config file were added from the flag + for (const configId in config.defaults.network.mainnet) { + const address = config.defaults.network.mainnet[configId]; + expect(address.host).toBeDefined(); + expect(address.port).toBeDefined(); + const foundConfig = await testBinUtils.pkStdio([ + 'nodes', + 'find', + configId, + '--node-path', + nodePath, + '--password-file', + passwordPath, + '--verbose', + ]); + expect(foundConfig.exitCode).toBe(0); + expect(foundConfig.stdout).toContain( + `Found node at ${address.host}:${address.port}`, + ); + } + + await testBinUtils.pkStdio( + [ + 'agent', + 'stop', + '--node-path', + nodePath, + '--password-file', + passwordPath, + ], + undefined, + dataDir, + ); + await status.waitFor('DEAD', 5000); + }, + global.defaultTimeout * 2, + ); + }); }); diff --git a/tests/bin/nodes.test.ts b/tests/bin/nodes.test.ts index f5fb736df..5b42f9b64 100644 --- a/tests/bin/nodes.test.ts +++ b/tests/bin/nodes.test.ts @@ -104,7 +104,7 @@ describe('CLI Nodes', () => { describe('commandClaimNode', () => { beforeAll(async () => { await remoteOnline.nodeManager.setNode(keynodeId, { - ip: polykeyAgent.revProxy.ingressHost, + host: polykeyAgent.revProxy.ingressHost, port: polykeyAgent.revProxy.ingressPort, } as NodeAddress); await polykeyAgent.acl.setNodePerm(remoteOnlineNodeId, { @@ -353,7 +353,7 @@ describe('CLI Nodes', () => { // Checking if node was added. const res = await polykeyAgent.nodeManager.getNode(validNodeId); expect(res).toBeTruthy(); - expect(res!.ip).toEqual(validHost); + expect(res!.host).toEqual(validHost); expect(res!.port).toEqual(port); }); test( diff --git a/tests/bin/notifications.test.ts b/tests/bin/notifications.test.ts index 3ccd1070f..a866b79d6 100644 --- a/tests/bin/notifications.test.ts +++ b/tests/bin/notifications.test.ts @@ -64,7 +64,7 @@ describe('CLI Notifications', () => { senderNodeId = senderPolykeyAgent.nodeManager.getNodeId(); receiverNodeId = receiverPolykeyAgent.nodeManager.getNodeId(); await senderPolykeyAgent.nodeManager.setNode(receiverNodeId, { - ip: receiverPolykeyAgent.revProxy.ingressHost, + host: receiverPolykeyAgent.revProxy.ingressHost, port: receiverPolykeyAgent.revProxy.ingressPort, } as NodeAddress); diff --git a/tests/bin/utils.ts b/tests/bin/utils.ts index d18be222b..8b8bf115b 100644 --- a/tests/bin/utils.ts +++ b/tests/bin/utils.ts @@ -43,6 +43,7 @@ async function pkStdio( }> { cwd = cwd ?? (await fs.promises.mkdtemp(path.join(os.tmpdir(), 'polykey-test-'))); + env['PK_SEED_NODES'] = env['PK_SEED_NODES'] ?? ''; // Parse the arguments of process.stdout.write and process.stderr.write const parseArgs = (args) => { const data = args[0]; @@ -131,6 +132,7 @@ async function pkExec( ...process.env, ...env, }; + env['PK_SEED_NODES'] = env['PK_SEED_NODES'] ?? ''; const tsConfigPath = path.resolve( path.join(global.projectDir, 'tsconfig.json'), ); @@ -195,6 +197,7 @@ async function pkSpawn( ...process.env, ...env, }; + env['PK_SEED_NODES'] = env['PK_SEED_NODES'] ?? ''; const tsConfigPath = path.resolve( path.join(global.projectDir, 'tsconfig.json'), ); @@ -257,6 +260,7 @@ async function pkExpect({ ...process.env, ...env, }; + env['PK_SEED_NODES'] = env['PK_SEED_NODES'] ?? ''; const tsConfigPath = path.resolve( path.join(global.projectDir, 'tsconfig.json'), ); diff --git a/tests/bin/vaults.test.ts b/tests/bin/vaults.test.ts index 1d8873a80..818e919a8 100644 --- a/tests/bin/vaults.test.ts +++ b/tests/bin/vaults.test.ts @@ -309,7 +309,7 @@ describe('CLI vaults', () => { const targetHost = targetPolykeyAgent.revProxy.ingressHost; const targetPort = targetPolykeyAgent.revProxy.ingressPort; await polykeyAgent.nodeManager.setNode(targetNodeId, { - ip: targetHost, + host: targetHost, port: targetPort, }); // Client agent: Start sending hole-punching packets to the target @@ -385,7 +385,7 @@ describe('CLI vaults', () => { const targetHost = targetPolykeyAgent.revProxy.ingressHost; const targetPort = targetPolykeyAgent.revProxy.ingressPort; await polykeyAgent.nodeManager.setNode(targetNodeId, { - ip: targetHost, + host: targetHost, port: targetPort, }); // Client agent: Start sending hole-punching packets to the target @@ -457,7 +457,7 @@ describe('CLI vaults', () => { const targetHost = targetPolykeyAgent.revProxy.ingressHost; const targetPort = targetPolykeyAgent.revProxy.ingressPort; await polykeyAgent.nodeManager.setNode(targetNodeId, { - ip: targetHost, + host: targetHost, port: targetPort, }); // Client agent: Start sending hole-punching packets to the target diff --git a/tests/client/rpcNodes.test.ts b/tests/client/rpcNodes.test.ts index fb5a616b0..13769ec6b 100644 --- a/tests/client/rpcNodes.test.ts +++ b/tests/client/rpcNodes.test.ts @@ -171,16 +171,16 @@ describe('Client service', () => { await nodesAdd(nodeAddressMessage, callCredentials); const nodeAddress = await nodeManager.getNode(nodeId); expect(nodeAddress).toBeDefined(); - expect(nodeAddress!.ip).toBe(host); + expect(nodeAddress!.host).toBe(host); expect(nodeAddress!.port).toBe(port); }); - test( + test.skip( 'should ping a node (online + offline)', async () => { const serverNodeId = polykeyServer.nodeManager.getNodeId(); await testKeynodeUtils.addRemoteDetails(polykeyAgent, polykeyServer); await polykeyServer.stop(); - const statusPath = path.join(polykeyServer.nodePath, 'status'); + const statusPath = path.join(polykeyServer.nodePath, 'status.json'); const status = new Status({ statusPath, fs, @@ -221,7 +221,7 @@ describe('Client service', () => { // Case 1: node already exists in the local node graph (no contact required) const nodeId = nodeId1; const nodeAddress: NodeAddress = { - ip: '127.0.0.1' as Host, + host: '127.0.0.1' as Host, port: 11111 as Port, }; await nodeManager.setNode(nodeId, nodeAddress); @@ -230,7 +230,7 @@ describe('Client service', () => { nodeMessage.setNodeId(nodeId); const res = await nodesFind(nodeMessage, callCredentials); expect(res.getNodeId()).toEqual(nodeId); - expect(res.getAddress()?.getHost()).toEqual(nodeAddress.ip); + expect(res.getAddress()?.getHost()).toEqual(nodeAddress.host); expect(res.getAddress()?.getPort()).toEqual(nodeAddress.port); }); test( @@ -240,7 +240,7 @@ describe('Client service', () => { // Case 2: node can be found on the remote node const nodeId = nodeId1; const nodeAddress: NodeAddress = { - ip: '127.0.0.1' as Host, + host: '127.0.0.1' as Host, port: 11111 as Port, }; // Setting the information on a remote node. @@ -253,7 +253,7 @@ describe('Client service', () => { nodeMessage.setNodeId(nodeId); const res = await nodesFind(nodeMessage, callCredentials); expect(res.getNodeId()).toEqual(nodeId); - expect(res.getAddress()?.getHost()).toEqual(nodeAddress.ip); + expect(res.getAddress()?.getHost()).toEqual(nodeAddress.host); expect(res.getAddress()?.getPort()).toEqual(nodeAddress.port); }, global.failedConnectionTimeout * 2, @@ -268,9 +268,9 @@ describe('Client service', () => { // Server will not be able to connect to this node (the only node in its // database), and will therefore not be able to locate the node. await polykeyServer.nodeManager.setNode(dummyNode, { - ip: '127.0.0.2' as Host, + host: '127.0.0.2' as Host, port: 22222 as Port, - } as NodeAddress); + }); const nodesFind = grpcUtils.promisifyUnaryCall( client, client.nodesFind, diff --git a/tests/client/rpcNotifications.test.ts b/tests/client/rpcNotifications.test.ts index 6c88724ac..ce9187334 100644 --- a/tests/client/rpcNotifications.test.ts +++ b/tests/client/rpcNotifications.test.ts @@ -1,5 +1,5 @@ import type * as grpc from '@grpc/grpc-js'; -import type { NodeInfo, NodeAddress } from '@/nodes/types'; +import type { NodeInfo } from '@/nodes/types'; import type { NodeManager } from '@/nodes'; import type { NotificationData } from '@/notifications/types'; import type { ClientServiceClient } from '@/proto/js/polykey/v1/client_service_grpc_pb'; @@ -123,9 +123,9 @@ describe('Notifications client service', () => { sender = await testKeynodeUtils.setupRemoteKeynode({ logger }); await sender.nodeManager.setNode(node1.id, { - ip: polykeyAgent.revProxy.ingressHost, + host: polykeyAgent.revProxy.ingressHost, port: polykeyAgent.revProxy.ingressPort, - } as NodeAddress); + }); await receiver.acl.setNodePerm(node1.id, { gestalt: { notify: null, diff --git a/tests/network/utils.test.ts b/tests/network/utils.test.ts index d71e27d59..3a73d9cb1 100644 --- a/tests/network/utils.test.ts +++ b/tests/network/utils.test.ts @@ -1,6 +1,6 @@ import type { Host, Port } from '@/network/types'; -import * as networkUtils from '@/network/utils'; +import { utils as networkUtils, errors as networkErrors } from '@/network'; describe('utils', () => { test('building addresses', async () => { @@ -23,4 +23,14 @@ describe('utils', () => { ), ).toBe('::1' as Host); }); + test('resolving hostnames', async () => { + await expect( + networkUtils.resolveHost('www.google.com' as Host), + ).resolves.toBeDefined(); + const host = await networkUtils.resolveHost('www.google.com' as Host); + expect(networkUtils.isValidHost(host)).toBeTruthy(); + await expect( + networkUtils.resolveHost('invalidHostname' as Host), + ).rejects.toThrow(networkErrors.ErrorHostnameResolutionFailed); + }); }); diff --git a/tests/nodes/NodeConnection.test.ts b/tests/nodes/NodeConnection.test.ts index 97b96cc75..3407119a0 100644 --- a/tests/nodes/NodeConnection.test.ts +++ b/tests/nodes/NodeConnection.test.ts @@ -340,6 +340,19 @@ describe('NodeConnection', () => { await conn.stop(); await conn.destroy(); }); + test('fails to connect to target (times out)', async () => { + await expect( + NodeConnection.createNodeConnection({ + targetNodeId: targetNodeId, + targetHost: '128.0.0.1' as Host, + targetPort: 12345 as Port, + connTimeout: 300, + forwardProxy: clientFwdProxy, + keyManager: clientKeyManager, + logger: logger, + }), + ).rejects.toThrow(nodesErrors.ErrorNodeConnectionTimeout); + }); test('receives 20 closest local nodes from connected target', async () => { const conn = await NodeConnection.createNodeConnection({ targetNodeId: targetNodeId, @@ -359,7 +372,7 @@ describe('NodeConnection', () => { i, ); const nodeAddress = { - ip: (i + '.' + i + '.' + i + '.' + i) as Host, + host: (i + '.' + i + '.' + i + '.' + i) as Host, port: i as Port, }; await serverNodeManager.setNode(closeNodeId, nodeAddress); @@ -373,7 +386,7 @@ describe('NodeConnection', () => { for (let i = 1; i <= 10; i++) { const farNodeId = nodeIdGenerator(i); const nodeAddress = { - ip: (i + '.' + i + '.' + i + '.' + i) as Host, + host: (i + '.' + i + '.' + i + '.' + i) as Host, port: i as Port, }; await serverNodeManager.setNode(farNodeId, nodeAddress); diff --git a/tests/nodes/NodeGraph.test.ts b/tests/nodes/NodeGraph.test.ts index 7ddb4b6c0..71c8ff1de 100644 --- a/tests/nodes/NodeGraph.test.ts +++ b/tests/nodes/NodeGraph.test.ts @@ -200,17 +200,17 @@ describe('NodeGraph', () => { test('finds correct node address', async () => { // New node added const newNode2Id = nodeId1; - const newNode2Address = { ip: '227.1.1.1', port: 4567 } as NodeAddress; + const newNode2Address = { host: '227.1.1.1', port: 4567 } as NodeAddress; await nodeGraph.setNode(newNode2Id, newNode2Address); // Get node address const foundAddress = await nodeGraph.getNode(newNode2Id); - expect(foundAddress).toEqual({ ip: '227.1.1.1', port: 4567 }); + expect(foundAddress).toEqual({ host: '227.1.1.1', port: 4567 }); }); test('unable to find node address', async () => { // New node added const newNode2Id = nodeId1; - const newNode2Address = { ip: '227.1.1.1', port: 4567 } as NodeAddress; + const newNode2Address = { host: '227.1.1.1', port: 4567 } as NodeAddress; await nodeGraph.setNode(newNode2Id, newNode2Address); // Get node address (of non-existent node) @@ -220,7 +220,7 @@ describe('NodeGraph', () => { test('adds a single node into a bucket', async () => { // New node added const newNode2Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 1); - const newNode2Address = { ip: '227.1.1.1', port: 4567 } as NodeAddress; + const newNode2Address = { host: '227.1.1.1', port: 4567 } as NodeAddress; await nodeGraph.setNode(newNode2Id, newNode2Address); // Check new node is in retrieved bucket from database @@ -228,36 +228,36 @@ describe('NodeGraph', () => { const bucket = await nodeGraph.getBucket(1); expect(bucket).toBeDefined(); expect(bucket![newNode2Id]).toEqual({ - address: { ip: '227.1.1.1', port: 4567 }, + address: { host: '227.1.1.1', port: 4567 }, lastUpdated: expect.any(Date), }); }); test('adds multiple nodes into the same bucket', async () => { // Add 3 new nodes into bucket 4 const newNode1Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 4); - const newNode1Address = { ip: '4.4.4.4', port: 4444 } as NodeAddress; + const newNode1Address = { host: '4.4.4.4', port: 4444 } as NodeAddress; await nodeGraph.setNode(newNode1Id, newNode1Address); const newNode2Id = nodesTestUtils.incrementNodeId(newNode1Id); - const newNode2Address = { ip: '5.5.5.5', port: 5555 } as NodeAddress; + const newNode2Address = { host: '5.5.5.5', port: 5555 } as NodeAddress; await nodeGraph.setNode(newNode2Id, newNode2Address); const newNode3Id = nodesTestUtils.incrementNodeId(newNode2Id); - const newNode3Address = { ip: '6.6.6.6', port: 6666 } as NodeAddress; + const newNode3Address = { host: '6.6.6.6', port: 6666 } as NodeAddress; await nodeGraph.setNode(newNode3Id, newNode3Address); // Based on XOR values, all 3 nodes should appear in bucket 4. const bucket = await nodeGraph.getBucket(4); if (bucket) { expect(bucket[newNode1Id]).toEqual({ - address: { ip: '4.4.4.4', port: 4444 }, + address: { host: '4.4.4.4', port: 4444 }, lastUpdated: expect.any(Date), }); expect(bucket[newNode2Id]).toEqual({ - address: { ip: '5.5.5.5', port: 5555 }, + address: { host: '5.5.5.5', port: 5555 }, lastUpdated: expect.any(Date), }); expect(bucket[newNode3Id]).toEqual({ - address: { ip: '6.6.6.6', port: 6666 }, + address: { host: '6.6.6.6', port: 6666 }, lastUpdated: expect.any(Date), }); } else { @@ -268,22 +268,22 @@ describe('NodeGraph', () => { test('adds a single node into different buckets', async () => { // New node for bucket 3 const newNode1Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 3); - const newNode1Address = { ip: '1.1.1.1', port: 1111 } as NodeAddress; + const newNode1Address = { host: '1.1.1.1', port: 1111 } as NodeAddress; await nodeGraph.setNode(newNode1Id, newNode1Address); // New node for bucket 255 (the highest possible bucket) const newNode2Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 255); - const newNode2Address = { ip: '2.2.2.2', port: 2222 } as NodeAddress; + const newNode2Address = { host: '2.2.2.2', port: 2222 } as NodeAddress; await nodeGraph.setNode(newNode2Id, newNode2Address); const bucket3 = await nodeGraph.getBucket(3); const bucket351 = await nodeGraph.getBucket(255); if (bucket3 && bucket351) { expect(bucket3[newNode1Id]).toEqual({ - address: { ip: '1.1.1.1', port: 1111 }, + address: { host: '1.1.1.1', port: 1111 }, lastUpdated: expect.any(Date), }); expect(bucket351[newNode2Id]).toEqual({ - address: { ip: '2.2.2.2', port: 2222 }, + address: { host: '2.2.2.2', port: 2222 }, lastUpdated: expect.any(Date), }); } else { @@ -294,14 +294,14 @@ describe('NodeGraph', () => { test('deletes a single node (and removes bucket)', async () => { // New node for bucket 2 const newNode1Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 2); - const newNode1Address = { ip: '4.4.4.4', port: 4444 } as NodeAddress; + const newNode1Address = { host: '4.4.4.4', port: 4444 } as NodeAddress; await nodeGraph.setNode(newNode1Id, newNode1Address); // Check the bucket is there first. const bucket = await nodeGraph.getBucket(2); if (bucket) { expect(bucket[newNode1Id]).toEqual({ - address: { ip: '4.4.4.4', port: 4444 }, + address: { host: '4.4.4.4', port: 4444 }, lastUpdated: expect.any(Date), }); } else { @@ -318,29 +318,29 @@ describe('NodeGraph', () => { test('deletes a single node (and retains remainder of bucket)', async () => { // Add 3 new nodes into bucket 4 const newNode1Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 4); - const newNode1Address = { ip: '4.4.4.4', port: 4444 } as NodeAddress; + const newNode1Address = { host: '4.4.4.4', port: 4444 } as NodeAddress; await nodeGraph.setNode(newNode1Id, newNode1Address); const newNode2Id = nodesTestUtils.incrementNodeId(newNode1Id); - const newNode2Address = { ip: '5.5.5.5', port: 5555 } as NodeAddress; + const newNode2Address = { host: '5.5.5.5', port: 5555 } as NodeAddress; await nodeGraph.setNode(newNode2Id, newNode2Address); const newNode3Id = nodesTestUtils.incrementNodeId(newNode2Id); - const newNode3Address = { ip: '6.6.6.6', port: 6666 } as NodeAddress; + const newNode3Address = { host: '6.6.6.6', port: 6666 } as NodeAddress; await nodeGraph.setNode(newNode3Id, newNode3Address); // Based on XOR values, all 3 nodes should appear in bucket 4. const bucket = await nodeGraph.getBucket(4); if (bucket) { expect(bucket[newNode1Id]).toEqual({ - address: { ip: '4.4.4.4', port: 4444 }, + address: { host: '4.4.4.4', port: 4444 }, lastUpdated: expect.any(Date), }); expect(bucket[newNode2Id]).toEqual({ - address: { ip: '5.5.5.5', port: 5555 }, + address: { host: '5.5.5.5', port: 5555 }, lastUpdated: expect.any(Date), }); expect(bucket[newNode3Id]).toEqual({ - address: { ip: '6.6.6.6', port: 6666 }, + address: { host: '6.6.6.6', port: 6666 }, lastUpdated: expect.any(Date), }); } else { @@ -355,11 +355,11 @@ describe('NodeGraph', () => { if (newBucket) { expect(newBucket[newNode1Id]).toBeUndefined(); expect(bucket[newNode2Id]).toEqual({ - address: { ip: '5.5.5.5', port: 5555 }, + address: { host: '5.5.5.5', port: 5555 }, lastUpdated: expect.any(Date), }); expect(bucket[newNode3Id]).toEqual({ - address: { ip: '6.6.6.6', port: 6666 }, + address: { host: '6.6.6.6', port: 6666 }, lastUpdated: expect.any(Date), }); } else { @@ -375,7 +375,7 @@ describe('NodeGraph', () => { for (let i = 1; i <= nodeGraph.maxNodesPerBucket; i++) { // Add the current node ID const nodeAddress = { - ip: (i + '.' + i + '.' + i + '.' + i) as Host, + host: (i + '.' + i + '.' + i + '.' + i) as Host, port: i as Port, }; await nodeGraph.setNode(currNodeId, nodeAddress); @@ -397,7 +397,7 @@ describe('NodeGraph', () => { // Attempt to add a new node into this full bucket (increment the last node // ID that was added) const newNodeId = nodesTestUtils.incrementNodeId(currNodeId); - const newNodeAddress = { ip: '0.0.0.1' as Host, port: 1234 as Port }; + const newNodeAddress = { host: '0.0.0.1' as Host, port: 1234 as Port }; await nodeGraph.setNode(newNodeId, newNodeAddress); const finalBucket = await nodeGraph.getBucket(59); @@ -420,29 +420,89 @@ describe('NodeGraph', () => { fail('Bucket undefined'); } }); + test('enforces k-bucket size, retaining all nodes if adding a pre-existing node', async () => { + // Add k nodes to the database (importantly, they all go into the same bucket) + let currNodeId = nodesTestUtils.generateNodeIdForBucket(nodeId, 59); + // Keep a record of the first node ID that we added + // const firstNodeId = currNodeId; + for (let i = 1; i <= nodeGraph.maxNodesPerBucket; i++) { + // Add the current node ID + const nodeAddress = { + host: (i + '.' + i + '.' + i + '.' + i) as Host, + port: i as Port, + }; + await nodeGraph.setNode(currNodeId, nodeAddress); + // Increment the current node ID - skip for the last one to keep currNodeId + // as the last added node ID + if (i !== nodeGraph.maxNodesPerBucket) { + const incrementedNodeId = nodesTestUtils.incrementNodeId(currNodeId); + currNodeId = incrementedNodeId; + } + } + // All of these nodes are in bucket 59 + const originalBucket = await nodeGraph.getBucket(59); + if (originalBucket) { + expect(Object.keys(originalBucket).length).toBe( + nodeGraph.maxNodesPerBucket, + ); + } else { + // Should be unreachable + fail('Bucket undefined'); + } + + // If we tried to re-add the first node, it would simply remove the original + // first node, as this is the "least active". + // We instead want to check that we don't mistakenly delete a node if we're + // updating an existing one. + // So, re-add the last node + const newLastAddress: NodeAddress = { + host: '30.30.30.30' as Host, + port: 30 as Port, + }; + await nodeGraph.setNode(currNodeId, newLastAddress); + + const finalBucket = await nodeGraph.getBucket(59); + if (finalBucket) { + // We should still have a full bucket + expect(Object.keys(finalBucket).length).toEqual( + nodeGraph.maxNodesPerBucket, + ); + // Ensure that this new node is in the bucket + expect(finalBucket[currNodeId]).toEqual({ + address: newLastAddress, + lastUpdated: expect.any(Date), + }); + } else { + // Should be unreachable + fail('Bucket undefined'); + } + }); test('retrieves all buckets (in expected lexicographic order)', async () => { // Bucket 0 is expected to never have any nodes (as nodeId XOR 0 = nodeId) // Bucket 1 (minimum): const node1Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 1); - const node1Address = { ip: '1.1.1.1', port: 1111 } as NodeAddress; + const node1Address = { host: '1.1.1.1', port: 1111 } as NodeAddress; await nodeGraph.setNode(node1Id, node1Address); // Bucket 4 (multiple nodes in 1 bucket): const node41Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 4); - const node41Address = { ip: '41.41.41.41', port: 4141 } as NodeAddress; + const node41Address = { host: '41.41.41.41', port: 4141 } as NodeAddress; await nodeGraph.setNode(node41Id, node41Address); const node42Id = nodesTestUtils.incrementNodeId(node41Id); - const node42Address = { ip: '42.42.42.42', port: 4242 } as NodeAddress; + const node42Address = { host: '42.42.42.42', port: 4242 } as NodeAddress; await nodeGraph.setNode(node42Id, node42Address); // Bucket 10 (lexicographic ordering - should appear after 2): const node10Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 10); - const node10Address = { ip: '10.10.10.10', port: 1010 } as NodeAddress; + const node10Address = { host: '10.10.10.10', port: 1010 } as NodeAddress; await nodeGraph.setNode(node10Id, node10Address); // Bucket 255 (maximum): const node255Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 255); - const node255Address = { ip: '255.255.255.255', port: 255 } as NodeAddress; + const node255Address = { + host: '255.255.255.255', + port: 255, + } as NodeAddress; await nodeGraph.setNode(node255Id, node255Address); const buckets = await nodeGraph.getAllBuckets(); @@ -452,29 +512,29 @@ describe('NodeGraph', () => { expect(buckets).toEqual([ { [node1Id]: { - address: { ip: '1.1.1.1', port: 1111 }, + address: { host: '1.1.1.1', port: 1111 }, lastUpdated: expect.any(String), }, }, { [node41Id]: { - address: { ip: '41.41.41.41', port: 4141 }, + address: { host: '41.41.41.41', port: 4141 }, lastUpdated: expect.any(String), }, [node42Id]: { - address: { ip: '42.42.42.42', port: 4242 }, + address: { host: '42.42.42.42', port: 4242 }, lastUpdated: expect.any(String), }, }, { [node10Id]: { - address: { ip: '10.10.10.10', port: 1010 }, + address: { host: '10.10.10.10', port: 1010 }, lastUpdated: expect.any(String), }, }, { [node255Id]: { - address: { ip: '255.255.255.255', port: 255 }, + address: { host: '255.255.255.255', port: 255 }, lastUpdated: expect.any(String), }, }, @@ -491,7 +551,7 @@ describe('NodeGraph', () => { i, ); const nodeAddress = { - ip: (i + '.' + i + '.' + i + '.' + i) as Host, + host: (i + '.' + i + '.' + i + '.' + i) as Host, port: i as Port, }; await nodeGraph.setNode(newNodeId, nodeAddress); @@ -544,7 +604,7 @@ describe('NodeGraph', () => { test('finds a single closest node', async () => { // New node added const newNode2Id = nodeId1; - const newNode2Address = { ip: '227.1.1.1', port: 4567 } as NodeAddress; + const newNode2Address = { host: '227.1.1.1', port: 4567 } as NodeAddress; await nodeGraph.setNode(newNode2Id, newNode2Address); // Find the closest nodes to some node, NODEID3 @@ -552,21 +612,21 @@ describe('NodeGraph', () => { expect(closest).toContainEqual({ id: newNode2Id, distance: 121n, - address: { ip: '227.1.1.1', port: 4567 }, + address: { host: '227.1.1.1', port: 4567 }, }); }); test('finds 3 closest nodes', async () => { // Add 3 nodes await nodeGraph.setNode(nodeId1, { - ip: '2.2.2.2', + host: '2.2.2.2', port: 2222, } as NodeAddress); await nodeGraph.setNode(nodeId2, { - ip: '3.3.3.3', + host: '3.3.3.3', port: 3333, } as NodeAddress); await nodeGraph.setNode(nodeId3, { - ip: '4.4.4.4', + host: '4.4.4.4', port: 4444, } as NodeAddress); @@ -576,17 +636,17 @@ describe('NodeGraph', () => { expect(closest).toContainEqual({ id: nodeId3, distance: 0n, - address: { ip: '4.4.4.4', port: 4444 }, + address: { host: '4.4.4.4', port: 4444 }, }); expect(closest).toContainEqual({ id: nodeId2, distance: 116n, - address: { ip: '3.3.3.3', port: 3333 }, + address: { host: '3.3.3.3', port: 3333 }, }); expect(closest).toContainEqual({ id: nodeId1, distance: 121n, - address: { ip: '2.2.2.2', port: 2222 }, + address: { host: '2.2.2.2', port: 2222 }, }); }); test('finds the 20 closest nodes', async () => { @@ -600,7 +660,7 @@ describe('NodeGraph', () => { i, ); const nodeAddress = { - ip: (i + '.' + i + '.' + i + '.' + i) as Host, + host: (i + '.' + i + '.' + i + '.' + i) as Host, port: i as Port, }; await nodeGraph.setNode(closeNodeId, nodeAddress); @@ -614,7 +674,7 @@ describe('NodeGraph', () => { for (let i = 1; i <= 10; i++) { const farNodeId = nodeIdGenerator(i); const nodeAddress = { - ip: (i + '.' + i + '.' + i + '.' + i) as Host, + host: (i + '.' + i + '.' + i + '.' + i) as Host, port: i as Port, }; await nodeGraph.setNode(farNodeId, nodeAddress); @@ -630,7 +690,7 @@ describe('NodeGraph', () => { test('updates node', async () => { // New node added const node1Id = nodesTestUtils.generateNodeIdForBucket(nodeId, 2); - const node1Address = { ip: '1.1.1.1', port: 1 } as NodeAddress; + const node1Address = { host: '1.1.1.1', port: 1 } as NodeAddress; await nodeGraph.setNode(node1Id, node1Address); // Check new node is in retrieved bucket from database @@ -638,7 +698,7 @@ describe('NodeGraph', () => { const time1 = bucket![node1Id].lastUpdated; // Update node and check that time is later - const newNode1Address = { ip: '2.2.2.2', port: 2 } as NodeAddress; + const newNode1Address = { host: '2.2.2.2', port: 2 } as NodeAddress; await nodeGraph.updateNode(node1Id, newNode1Address); const bucket2 = await nodeGraph.getBucket(2); diff --git a/tests/nodes/NodeManager.test.ts b/tests/nodes/NodeManager.test.ts index a7b41484e..060bb6fc0 100644 --- a/tests/nodes/NodeManager.test.ts +++ b/tests/nodes/NodeManager.test.ts @@ -158,7 +158,7 @@ describe('NodeManager', () => { await target.start({ password: 'password' }); targetNodeId = target.keyManager.getNodeId(); targetNodeAddress = { - ip: target.revProxy.ingressHost, + host: target.revProxy.ingressHost, port: target.revProxy.ingressPort, }; await nodeManager.setNode(targetNodeId, targetNodeAddress); @@ -224,7 +224,7 @@ describe('NodeManager', () => { async () => { // Add the dummy node await nodeManager.setNode(dummyNode, { - ip: '125.0.0.1' as Host, + host: '125.0.0.1' as Host, port: 55555 as Port, }); // @ts-ignore accessing protected NodeConnectionMap @@ -259,7 +259,7 @@ describe('NodeManager', () => { }); const serverNodeId = server.nodeManager.getNodeId(); let serverNodeAddress: NodeAddress = { - ip: server.revProxy.ingressHost, + host: server.revProxy.ingressHost, port: server.revProxy.ingressPort, }; await nodeManager.setNode(serverNodeId, serverNodeAddress); @@ -274,7 +274,7 @@ describe('NodeManager', () => { await server.start({ password: 'password' }); // Update the node address (only changes because we start and stop) serverNodeAddress = { - ip: server.revProxy.ingressHost, + host: server.revProxy.ingressHost, port: server.revProxy.ingressPort, }; await nodeManager.setNode(serverNodeId, serverNodeAddress); @@ -301,7 +301,7 @@ describe('NodeManager', () => { // Case 1: node already exists in the local node graph (no contact required) const nodeId = nodeId1; const nodeAddress: NodeAddress = { - ip: '127.0.0.1' as Host, + host: '127.0.0.1' as Host, port: 11111 as Port, }; await nodeManager.setNode(nodeId, nodeAddress); @@ -316,12 +316,12 @@ describe('NodeManager', () => { // Case 2: node can be found on the remote node const nodeId = nodeId1; const nodeAddress: NodeAddress = { - ip: '127.0.0.1' as Host, + host: '127.0.0.1' as Host, port: 11111 as Port, }; const server = await testUtils.setupRemoteKeynode({ logger: logger }); await nodeManager.setNode(server.nodeManager.getNodeId(), { - ip: server.revProxy.ingressHost, + host: server.revProxy.ingressHost, port: server.revProxy.ingressPort, } as NodeAddress); await server.nodeManager.setNode(nodeId, nodeAddress); @@ -339,14 +339,14 @@ describe('NodeManager', () => { const nodeId = nodeId1; const server = await testUtils.setupRemoteKeynode({ logger: logger }); await nodeManager.setNode(server.nodeManager.getNodeId(), { - ip: server.revProxy.ingressHost, + host: server.revProxy.ingressHost, port: server.revProxy.ingressPort, } as NodeAddress); // Add a dummy node to the server node graph database // Server will not be able to connect to this node (the only node in its // database), and will therefore not be able to locate the node. await server.nodeManager.setNode(dummyNode, { - ip: '127.0.0.2' as Host, + host: '127.0.0.2' as Host, port: 22222 as Port, } as NodeAddress); // So unfindableNode cannot be found @@ -361,7 +361,7 @@ describe('NodeManager', () => { test('knows node (true and false case)', async () => { // Known node const nodeAddress1: NodeAddress = { - ip: '127.0.0.1' as Host, + host: '127.0.0.1' as Host, port: 11111 as Port, }; await nodeManager.setNode(nodeId1, nodeAddress1); @@ -396,7 +396,7 @@ describe('NodeManager', () => { }); xNodeId = x.nodeManager.getNodeId(); xNodeAddress = { - ip: x.revProxy.ingressHost, + host: x.revProxy.ingressHost, port: x.revProxy.ingressPort, }; xPublicKey = x.keyManager.getRootKeyPairPem().publicKey; @@ -406,7 +406,7 @@ describe('NodeManager', () => { }); yNodeId = y.nodeManager.getNodeId(); yNodeAddress = { - ip: y.revProxy.ingressHost, + host: y.revProxy.ingressHost, port: y.revProxy.ingressPort, }; yPublicKey = y.keyManager.getRootKeyPairPem().publicKey; diff --git a/tests/notifications/NotificationsManager.test.ts b/tests/notifications/NotificationsManager.test.ts index 025ef10ca..6ef1a048a 100644 --- a/tests/notifications/NotificationsManager.test.ts +++ b/tests/notifications/NotificationsManager.test.ts @@ -249,7 +249,7 @@ describe('NotificationsManager', () => { }); await senderNodeManager.start(); await senderNodeManager.setNode(receiverNodeId, { - ip: receiverHost, + host: receiverHost, port: receiverIngressPort, } as NodeAddress); diff --git a/tests/utils.ts b/tests/utils.ts index a3bd2027f..65b6854e5 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -67,7 +67,7 @@ async function addRemoteDetails( ) { // Add remote node's details to local node await localNode.nodeManager.setNode(remoteNode.nodeManager.getNodeId(), { - ip: remoteNode.revProxy.ingressHost, + host: remoteNode.revProxy.ingressHost, port: remoteNode.revProxy.ingressPort, } as NodeAddress); } diff --git a/tests/vaults/VaultManager.test.ts b/tests/vaults/VaultManager.test.ts index df9537c70..f7db6f8a1 100644 --- a/tests/vaults/VaultManager.test.ts +++ b/tests/vaults/VaultManager.test.ts @@ -818,7 +818,7 @@ describe('VaultManager', () => { await vaultOps.addSecret(vault, name, content); } await nodeManager.setNode(targetNodeId, { - ip: targetHost, + host: targetHost, port: targetPort, } as NodeAddress); await nodeManager.getConnectionToNode(targetNodeId);