diff --git a/src/client/utils/ajax/ajax.ts b/src/client/utils/ajax/ajax.ts index 96728c111..1b8a8e75d 100644 --- a/src/client/utils/ajax/ajax.ts +++ b/src/client/utils/ajax/ajax.ts @@ -18,8 +18,9 @@ import axios from "axios"; import { Dataset, DatasetJS, Environment, Executor, Expression } from "plywood"; import { ClientAppSettings } from "../../../common/models/app-settings/app-settings"; +import { SerializedDataCube } from "../../../common/models/data-cube/data-cube"; import { isEnabled, Oauth } from "../../../common/models/oauth/oauth"; -import { ClientSources, SerializedSources } from "../../../common/models/sources/sources"; +import { ClientSources } from "../../../common/models/sources/sources"; import { deserialize } from "../../deserializers/sources"; import { getToken, mapOauthError } from "../../oauth/oauth"; @@ -54,12 +55,12 @@ export class Ajax { const headers = Ajax.headers(oauth); return axios({ method, url, data, timeout, validateStatus, headers }) - .then(res => { - return res.data; - }) - .catch(error => { - throw mapOauthError(oauth, error); - }); + .then(res => { + return res.data; + }) + .catch(error => { + throw mapOauthError(oauth, error); + }); } static queryUrlExecutorFactory(dataCubeName: string, { oauth, clientTimeout: timeout }: ClientAppSettings): Executor { @@ -69,17 +70,34 @@ export class Ajax { const timezone = env ? env.timezone : null; const data = { dataCube: dataCubeName, expression: ex.toJS(), timezone }; return Ajax.query<{ result: DatasetJS }>({ method, url, timeout, data }, oauth) - .then(res => Dataset.fromJS(res.result)); + .then(res => Dataset.fromJS(res.result)); }; } - static sources(appSettings: ClientAppSettings): Promise { - const headers = Ajax.headers(appSettings.oauth); - return axios.get("sources", { headers }) - .then(resp => resp.data) - .catch(error => { - throw mapOauthError(appSettings.oauth, error); - }) - .then(sourcesJS => deserialize(sourcesJS, appSettings)); + static async sources(appSettings: ClientAppSettings): Promise { + try { + const headers = Ajax.headers(appSettings.oauth); + const [clusters, dataCubesResult] = await Promise.all([this.fetchClusters(headers), this.fetchDataCubes(headers)]); + return deserialize({ clusters, dataCubes: dataCubesResult }, appSettings); + } catch (e) { + throw mapOauthError(appSettings.oauth, e); + } + } + + private static async fetchDataCubes(headers: {}) { + let dataCubesResult: SerializedDataCube[] = []; + let isDone = false; + let page = 0; + while (!isDone) { + const { dataCubes, isDone: isDoneResult } = (await axios.get(`sources/dataCubes?page=${page}`, { headers })).data; + dataCubesResult = [...dataCubesResult, ...dataCubes]; + isDone = isDoneResult; + page += 1; + } + return dataCubesResult; + } + + private static async fetchClusters(headers: {}) { + return (await axios.get("sources/clusters", { headers })).data; } } diff --git a/src/common/models/sources/sources.ts b/src/common/models/sources/sources.ts index cf46006b8..df44d79e3 100644 --- a/src/common/models/sources/sources.ts +++ b/src/common/models/sources/sources.ts @@ -22,7 +22,6 @@ import { Cluster, ClusterJS, fromConfig as clusterFromConfig, - serialize as serializeCluster, SerializedCluster } from "../cluster/cluster"; import { findCluster } from "../cluster/find-cluster"; @@ -96,20 +95,10 @@ export function fromConfig(config: SourcesJS, logger: Logger): Sources { }; } -export function serialize({ - clusters: serverClusters, - dataCubes: serverDataCubes - }: Sources): SerializedSources { - const clusters = serverClusters.map(serializeCluster); - - const dataCubes = serverDataCubes - .filter(dc => isQueryable(dc)) - .map(serializeDataCube); - - return { - clusters, - dataCubes - }; +export function serializeDataCubes(dataCubes: DataCube[]) { + return dataCubes + .filter(isQueryable) + .map(serializeDataCube); } export function getDataCubesForCluster(sources: Sources, clusterName: string): DataCube[] { diff --git a/src/server/routes/sources/sources.ts b/src/server/routes/sources/sources.ts index fd62ef964..59ff3f7fa 100644 --- a/src/server/routes/sources/sources.ts +++ b/src/server/routes/sources/sources.ts @@ -16,33 +16,54 @@ import { Request, Response, Router } from "express"; import { errorToMessage } from "../../../common/logger/logger"; -import { serialize } from "../../../common/models/sources/sources"; +import { serialize as serializeCluster } from "../../../common/models/cluster/cluster"; +import { serializeDataCubes } from "../../../common/models/sources/sources"; import { checkAccess } from "../../utils/datacube-guard/datacube-guard"; import { SettingsManager } from "../../utils/settings-manager/settings-manager"; export function sourcesRouter(settings: Pick) { - const logger = settings.logger.setLoggerId("Sources"); + const logger = settings.logger.setLoggerId("Sources"); - const router = Router(); + const router = Router(); + const MAX_DATA_CUBES_IN_REQUEST = 1000; + router.get("/clusters", async (req: Request, res: Response) => { + try { + const { clusters } = await settings.getSources(); + res.json(clusters.map(serializeCluster)); + } catch (error) { + logger.error(errorToMessage(error)); - router.get("/", async (req: Request, res: Response) => { + res.status(500).send({ + error: "Can't fetch settings", + message: error.message + }); + } + }); + router.get("/dataCubes", async (req: Request, res: Response) => { + try { + const { dataCubes } = await settings.getSources(); + const relevantDataCubes = serializeDataCubes(dataCubes.filter(dataCube => checkAccess(dataCube, req.headers))); + if (relevantDataCubes.length < MAX_DATA_CUBES_IN_REQUEST) { + res.json({ dataCubes: relevantDataCubes, isDone: true }); + } else { + const currentPageNumber = (req.query["page"] && parseInt(req.query["page"] as string, 10)) || 0; + const dataSourcesStart = currentPageNumber * MAX_DATA_CUBES_IN_REQUEST; + const dataSourcesEnd = dataSourcesStart + MAX_DATA_CUBES_IN_REQUEST; + const isDone = dataSourcesEnd >= dataCubes.length; + res.json({ + dataCubes: relevantDataCubes.slice(dataSourcesStart, dataSourcesEnd), + isDone + }); + } + } catch (error) { + logger.error(errorToMessage(error)); + res.status(500).send({ + error: "Can't fetch settings", + message: error.message + }); + } + }); - try { - const { clusters, dataCubes } = await settings.getSources(); - res.json(serialize({ - clusters, - dataCubes: dataCubes.filter( dataCube => checkAccess(dataCube, req.headers) ) - })); - } catch (error) { - logger.error(errorToMessage(error)); - - res.status(500).send({ - error: "Can't fetch settings", - message: error.message - }); - } - }); - - return router; + return router; }