Merge pull request #12354 from Budibase/fix/12154-invalid-table-fetches

Fix issues with data UI loading some tables

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -109,14 +109,36 @@ export class DatabaseImpl implements Database {
  }
  }
 
- async get<T>(id?: string): Promise<T | any> {
+ async get<T extends Document>(id?: string): Promise<T> {
  const db = await this.checkSetup()
  if (!id) {
  throw new Error("Unable to get doc without a valid _id.")
  }
  return this.updateOutput(() => db.get(id))
  }
 
+ async getMultiple<T extends Document>(
+ ids: string[],
+ opts?: { allowMissing?: boolean }
+ ): Promise<T[]> {
+ // get unique
+ ids = [...new Set(ids)]
+ const response = await this.allDocs<T>({
+ keys: ids,
+ include_docs: true,
+ })
+ const NOT_FOUND = "not_found"
+ const rows = response.rows.filter(row => row.error !== NOT_FOUND)
+ const someMissing = rows.length !== response.rows.length
+ // some were filtered out - means some missing
+ if (!opts?.allowMissing && someMissing) {
+ const missing = response.rows.filter(row => row.error === NOT_FOUND)
+ const missingIds = missing.map(row => row.key).join(", ")
+ throw new Error(`Unable to get documents: ${missingIds}`)
+ }
+ return rows.map(row => row.doc!)
+ }
+
  async remove(idOrDoc: string | Document, rev?: string) {
  const db = await this.checkSetup()
  let _id: string

packages/server/src/api/controllers/row/internal.ts

@@ -1,9 +1,5 @@
 import * as linkRows from "../../../db/linkedRows"
-import {
- generateRowID,
- getMultiIDParams,
- InternalTables,
-} from "../../../db/utils"
+import { generateRowID, InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import {
  cleanupAttachments,
@@ -240,8 +236,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
  const linkVals = links as LinkDocumentValue[]
 
  // look up the actual rows based on the ids
- const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
- let linkedRows = (await db.allDocs<Row>(params)).rows.map(row => row.doc!)
+ let linkedRows = await db.getMultiple<Row>(
+ linkVals.map(linkVal => linkVal.id),
+ { allowMissing: true }
+ )
 
  // get the linked tables
  const linkTableIds = getLinkedTableIDs(table as Table)

packages/server/src/api/controllers/row/utils.ts

@@ -1,21 +1,9 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import { context } from "@budibase/backend-core"
-import {
- Ctx,
- FieldType,
- ManyToOneRelationshipFieldMetadata,
- OneToManyRelationshipFieldMetadata,
- Row,
- SearchFilters,
- Table,
- UserCtx,
-} from "@budibase/types"
-import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
-import sdk from "../../../sdk"
+import { Ctx, Row, UserCtx } from "@budibase/types"
 
 import validateJs from "validate.js"
-import { cloneDeep } from "lodash/fp"
 
 validateJs.extend(validateJs.validators.datetime, {
  parse: function (value: string) {

packages/server/src/db/linkedRows/index.ts

@@ -8,7 +8,7 @@ import {
  getLinkedTable,
 } from "./linkUtils"
 import flatten from "lodash/flatten"
-import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
+import { USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
@@ -79,9 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
  const db = context.getAppDB()
  const linkedRowIds = links.map(link => link.id)
  const uniqueRowIds = [...new Set(linkedRowIds)]
- let dbRows = (await db.allDocs<Row>(getMultiIDParams(uniqueRowIds))).rows.map(
- row => row.doc!
- )
+ let dbRows = await db.getMultiple<Row>(uniqueRowIds, { allowMissing: true })
  // convert the unique db rows back to a full list of linked rows
  const linked = linkedRowIds
  .map(id => dbRows.find(row => row && row._id === id))

packages/server/src/db/utils.ts

@@ -283,16 +283,6 @@ export function generatePluginID(name: string) {
  return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
 }
 
-/**
- * This can be used with the db.allDocs to get a list of IDs
- */
-export function getMultiIDParams(ids: string[]) {
- return {
- keys: ids,
- include_docs: true,
- }
-}
-
 /**
  * Generates a new view ID.
  * @returns The new view ID which the view doc can be stored under.

packages/server/src/sdk/app/tables/getters.ts

@@ -1,5 +1,5 @@
 import { context } from "@budibase/backend-core"
-import { getMultiIDParams, getTableParams } from "../../../db/utils"
+import { getTableParams } from "../../../db/utils"
 import {
  breakExternalTableId,
  isExternalTableID,
@@ -17,6 +17,9 @@ import datasources from "../datasources"
 import sdk from "../../../sdk"
 
 export function processTable(table: Table): Table {
+ if (!table) {
+ return table
+ }
  if (table._id && isExternalTableID(table._id)) {
  return {
  ...table,
@@ -73,6 +76,9 @@ export async function getExternalTable(
  tableName: string
 ): Promise<Table> {
  const entities = await getExternalTablesInDatasource(datasourceId)
+ if (!entities[tableName]) {
+ throw new Error(`Unable to find table named "${tableName}"`)
+ }
  return processTable(entities[tableName])
 }
 
@@ -124,10 +130,10 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
  }
  if (internalTableIds.length) {
  const db = context.getAppDB()
- const internalTableDocs = await db.allDocs<Table>(
- getMultiIDParams(internalTableIds)
- )
- tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))
+ const internalTables = await db.getMultiple<Table>(internalTableIds, {
+ allowMissing: true,
+ })
+ tables = tables.concat(internalTables)
  }
  return processTables(tables)
 }

packages/server/src/sdk/tests/tables.spec.ts

@@ -0,0 +1,39 @@
+import TestConfig from "../../tests/utilities/TestConfiguration"
+import { basicTable } from "../../tests/utilities/structures"
+import { Table } from "@budibase/types"
+import sdk from "../"
+
+describe("tables", () => {
+ const config = new TestConfig()
+ let table: Table
+
+ beforeAll(async () => {
+ await config.init()
+ table = await config.api.table.create(basicTable())
+ })
+
+ describe("getTables", () => {
+ it("should be able to retrieve tables", async () => {
+ await config.doInContext(config.appId, async () => {
+ const tables = await sdk.tables.getTables([table._id!])
+ expect(tables.length).toBe(1)
+ expect(tables[0]._id).toBe(table._id)
+ expect(tables[0].name).toBe(table.name)
+ })
+ })
+
+ it("shouldn't fail when retrieving tables that don't exist", async () => {
+ await config.doInContext(config.appId, async () => {
+ const tables = await sdk.tables.getTables(["unknown"])
+ expect(tables.length).toBe(0)
+ })
+ })
+
+ it("should de-duplicate the IDs", async () => {
+ await config.doInContext(config.appId, async () => {
+ const tables = await sdk.tables.getTables([table._id!, table._id!])
+ expect(tables.length).toBe(1)
+ })
+ })
+ })
+})

packages/server/src/utilities/global.ts

@@ -1,4 +1,4 @@
-import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils"
+import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import {
  roles,
  db as dbCore,
@@ -96,9 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
  const db = tenancy.getGlobalDB()
  let globalUsers: User[]
  if (userIds) {
- globalUsers = (await db.allDocs<User>(getMultiIDParams(userIds))).rows.map(
- row => row.doc!
- )
+ globalUsers = await db.getMultiple<User>(userIds, { allowMissing: true })
  } else {
  globalUsers = (
  await db.allDocs<User>(

packages/types/src/sdk/db.ts

@@ -122,7 +122,11 @@ export interface Database {
 
  exists(): Promise<boolean>
  checkSetup(): Promise<Nano.DocumentScope<any>>
- get<T>(id?: string): Promise<T>
+ get<T extends Document>(id?: string): Promise<T>
+ getMultiple<T extends Document>(
+ ids: string[],
+ opts?: { allowMissing?: boolean }
+ ): Promise<T[]>
  remove(
  id: string | Document,
  rev?: string