Merge pull request #226 from Digital-Engineering/development

Development

API Documentation/Core.swagger_collection.yaml

@@ -3772,6 +3772,12 @@ paths:
  file:
  type: string
  format: binary
+ parameters:
+ - schema:
+ type: boolean
+ default: false
+ in: query
+ name: isEnabled
  '/containers/{container_id}/import/datasources/{data_source_id}/mappings/{mapping_id}/transformations':
  get:
  tags:

Dockerfile

@@ -2,6 +2,7 @@ FROM cimg/rust:1.65.0-node as build
 
 USER root
 RUN sudo apt-get update
+RUN sudo apt-get upgrade libtasn1-6
 
 # these settings are needed for the admin web gui build, these variables are all baked into the Vue application and thus
 # are available to any end user that wants to dig deep enough in the webpage - as such we don't feel it a security risk
@@ -43,7 +44,7 @@ RUN npm run build:web-gl
 # catch any env file a user might have accidentally built into the container
 RUN rm -rf .env
 
-FROM node:18.13-buster-slim as production
+FROM node:18.14.0-buster-slim as production
 RUN apt-get update
 
 WORKDIR /srv/core_api

package.json

@@ -139,7 +139,7 @@
  "ioredis": "^4.24.2",
  "isomorphic-dompurify": "^0.15.0",
  "JSONStream": "^1.3.5",
- "jsonwebtoken": "^8.5.1",
+ "jsonwebtoken": "^9.0.0",
  "kind-of": "^6.0.3",
  "method-override": "^3.0.0",
  "minimist": "^1.2.5",

src/data_access_layer/mappers/data_warehouse/data/edge_mapper.ts

@@ -144,7 +144,7 @@ export default class EdgeMapper extends Mapper {
  ON CONFLICT(container_id,relationship_pair_id,data_source_id,created_at, origin_id, destination_id) DO UPDATE SET
  properties = EXCLUDED.properties,
  metadata = EXCLUDED.metadata
- WHERE EXCLUDED.id = edges.id
+ WHERE EXCLUDED.id = edges.id AND excluded.properties IS DISTINCT FROM edges.properties
  RETURNING *`;
 
  const values = edges.map((e) => [
@@ -236,7 +236,7 @@ export default class EdgeMapper extends Mapper {
  LEFT JOIN metatype_relationships ON metatype_relationship_pairs.relationship_id = metatype_relationships.id
  WHERE edges.id = $1 ORDER BY edges.created_at ASC`,
  values: [edgeID],
- }
+ };
  }
 
  private retrieveRawDataHistoryStatement(edgeID: string): QueryConfig {
@@ -247,7 +247,7 @@ export default class EdgeMapper extends Mapper {
  LEFT JOIN data_staging ON edges.data_staging_id = data_staging.id
  WHERE edges.id = $1 ORDER BY edges.created_at ASC`,
  values: [edgeID],
- }
+ };
  }
 
  private retrieveByRelationshipStatement(origin: string, relationship: string, destination: string): QueryConfig {

src/data_access_layer/mappers/data_warehouse/data/node_mapper.ts

@@ -166,7 +166,7 @@ export default class NodeMapper extends Mapper {
  properties = EXCLUDED.properties,
  metadata = EXCLUDED.metadata,
  deleted_at = EXCLUDED.deleted_at
- WHERE EXCLUDED.id = nodes.id 
+ WHERE EXCLUDED.id = nodes.id AND EXCLUDED.properties IS DISTINCT FROM nodes.properties
  RETURNING *`;
 
  const values = nodes.map((n) => [
@@ -237,7 +237,7 @@ export default class NodeMapper extends Mapper {
  FROM nodes LEFT JOIN metatypes ON nodes.metatype_id = metatypes.id
  WHERE nodes.id = $1 ORDER BY nodes.created_at ASC`,
  values: [nodeID],
- }
+ };
  }
 
  // retrieves node history with raw data records attached
@@ -248,7 +248,7 @@ export default class NodeMapper extends Mapper {
  LEFT JOIN data_staging ON nodes.data_staging_id = data_staging.id
  WHERE nodes.id = $1 ORDER BY nodes.created_at ASC`,
  values: [nodeID],
- }
+ };
  }
 
  private domainRetrieveStatement(nodeID: string, containerID: string): QueryConfig {

src/data_access_layer/migrations/059_node_insert_trigger_rewrite.sql

@@ -0,0 +1,32 @@
+CREATE OR REPLACE FUNCTION node_insert_trigger() RETURNS TRIGGER AS $$
+DECLARE
+ old_id bigint;
+ old_properties jsonb;
+BEGIN
+ IF NEW.original_data_id IS NOT NULL THEN
+ BEGIN
+ SELECT nodes.id, nodes.properties
+ INTO old_id, old_properties
+ FROM nodes
+ WHERE original_data_id = NEW.original_data_id
+ AND metatype_id = NEW.metatype_id
+ AND data_source_id = NEW.data_source_id LIMIT 1;
+ EXCEPTION
+ WHEN NO_DATA_FOUND THEN
+ old_id := NULL;
+ END;
+
+ IF old_id IS NOT NULL THEN
+ NEW.id = old_id;
+/*
+ if the old properties are exactly the same as the new properties, silently discard the insert
+ */
+ IF old_properties IS NOT DISTINCT FROM NEW.properties THEN
+ RETURN NULL;
+ END IF;
+ END IF;
+ END IF;
+
+ RETURN NEW;
+END;
+$$ language plpgsql;

src/data_access_layer/repositories/access_management/oauth_repository.ts

@@ -178,7 +178,7 @@ export default class OAuthRepository extends Repository implements RepositoryInt
  if (!valid) return new Promise((resolve) => resolve(Result.Failure('invalid client')));
  }
 
- const token = jwt.sign(classToPlain(UserToReturnUser(user.value)), Config.encryption_key_secret, {expiresIn: '720m', algorithm: 'RS256'});
+ const token = jwt.sign(classToPlain(UserToReturnUser(user.value)), Config.encryption_key_secret, {expiresIn: '720m', algorithm: 'RS256', allowInsecureKeySizes: true});
 
  return new Promise((resolve) => resolve(Result.Success(token)));
  }

src/data_access_layer/repositories/data_warehouse/etl/type_mapping_repository.ts

@@ -326,7 +326,7 @@ export default class TypeMappingRepository extends Repository implements Reposit
  // by name instead of uuid - so there is potential for issues, use with caution. We return the newly modified/created
  // type mappings as well as failed mappings so that the end user can perform a review of the export - check the value
  // of isError on the return to determine if import was successful
- async importToDataSource(targetSourceID: string, user: User, ...originalMappings: TypeMapping[]): Promise<Result<TypeMapping>[]> {
+ async importToDataSource(targetSourceID: string, user: User, active: boolean, ...originalMappings: TypeMapping[]): Promise<Result<TypeMapping>[]> {
  // pull in the target data source, immediately error out if the source isn't valid, we also need it for the container
  // in this case we're using the data source mapper because we have no need of actually performing any operations
  const targetDataSource = await this.#dataSourceMapper.Retrieve(targetSourceID);
@@ -355,7 +355,7 @@ export default class TypeMappingRepository extends Repository implements Reposit
  // call the repo's save method on the modified mapping.
  mapping.data_source_id = targetDataSource.value.id;
  mapping.container_id = targetDataSource.value.container_id;
- mapping.active = false; // always inactivate the mapping after modification
+ mapping.active = active; 
 
  // now we must iterate through the transformations and potentially back-fill the metatype/relationship
  // ids and key ids if all that are present are the names - note that this will not modify the mapping if

src/http_server/routes/access_management/oauth_routes.ts

@@ -630,7 +630,7 @@ export default class OAuthRoutes {
  }
 
  try {
- const token = jwt.sign(classToPlain(user.value), Config.encryption_key_secret, {expiresIn: expiry, algorithm: 'RS256'});
+ const token = jwt.sign(classToPlain(user.value), Config.encryption_key_secret, {expiresIn: expiry, algorithm: 'RS256', allowInsecureKeySizes: true});
  res.status(200).json(token);
  return;
  } catch (e: any) {

src/http_server/routes/data_warehouse/etl/type_mapping_routes.ts

@@ -9,6 +9,8 @@ import TypeTransformation from '../../../../domain_objects/data_warehouse/etl/ty
 import TypeTransformationRepository from '../../../../data_access_layer/repositories/data_warehouse/etl/type_transformation_repository';
 import TypeMapping, {TypeMappingExportPayload, TypeMappingUpgradePayload} from '../../../../domain_objects/data_warehouse/etl/type_mapping';
 import {FileInfo} from 'busboy';
+import DataSourceRecord from '../../../../domain_objects/data_warehouse/import/data_source';
+import { none } from 'fp-ts/lib/Option';
 
 const JSONStream = require('JSONStream');
 const Busboy = require('busboy');
@@ -199,7 +201,7 @@ export default class TypeMappingRoutes {
  });
  } else {
  mappingRepo
- .importToDataSource(payload.target_data_source, user, ...results.value)
+ .importToDataSource(payload.target_data_source, user, false, ...results.value)
  .then((result) => {
  res.status(200).json(result);
  next();
@@ -426,6 +428,13 @@ export default class TypeMappingRoutes {
  private static importTypeMappings(req: Request, res: Response, next: NextFunction) {
  const user = req.currentUser!;
  const importResults: Promise<Result<TypeMapping>[]>[] = [];
+ let active : boolean;
+
+ if(req.query['isEnabled' as keyof object] === 'true') {
+ active = true;
+ } else {
+ active = false;
+ }
 
  if (!req.dataSource) {
  Result.Failure(`unable to find data source`, 404).asResponse(res);
@@ -437,8 +446,8 @@ export default class TypeMappingRoutes {
  if (req.headers['content-type']?.includes('application/json')) {
  const repo = new TypeMappingRepository();
  const payload = plainToClass(TypeMapping, req.body);
-
- repo.importToDataSource(req.dataSource.DataSourceRecord?.id!, user, ...payload)
+ 
+ repo.importToDataSource(req.dataSource.DataSourceRecord?.id!, user, active, ...payload)
  .then((results) => {
  res.status(201).json(results);
  next();
@@ -468,11 +477,11 @@ export default class TypeMappingRoutes {
  Result.Failure(e).asResponse(res);
  return;
  });
-
+ 
  // once the file has been read, convert to mappings and then attempt the import
  stream.on('end', () => {
  const mappings = plainToClass(TypeMapping, objects);
- importResults.push(repo.importToDataSource(req.dataSource?.DataSourceRecord?.id!, user, ...mappings));
+ importResults.push(repo.importToDataSource(req.dataSource?.DataSourceRecord?.id!, user, active, ...mappings));
  });
 
  try {

src/tests/data_warehouse/etl/type_mapping_repository.spec.ts

@@ -35,6 +35,7 @@ describe('A Type Mapping Repository', async () => {
  let containerID: string = process.env.TEST_CONTAINER_ID || '';
  let dataSourceID: string;
  let targetDataSourceID: string;
+ let active: boolean;
  let user: User;
  let metatype: Metatype;
  let key: MetatypeKey;
@@ -172,6 +173,7 @@ describe('A Type Mapping Repository', async () => {
  expect(exp2.isError).false;
  expect(exp2.value).not.empty;
  targetDataSourceID = exp2.value.id!;
+ active = exp2.value.active;
 
  // create the data source in the new container for mapping/transformation import/export tests
  const exp3 = await DataSourceMapper.Instance.Create(
@@ -305,7 +307,7 @@ describe('A Type Mapping Repository', async () => {
  expect(mapping.transformations![0]!.id).not.undefined;
 
  // first we attempt to export them into the same container but separate data source
- let exported = await repo.importToDataSource(targetDataSourceID, user, mapping);
+ let exported = await repo.importToDataSource(targetDataSourceID, user, active, mapping);
  for (const result of exported) {
  expect(result.isError).false;
  expect(result.value.id).not.eq(mapping.id); // should be a new mapping
@@ -322,7 +324,7 @@ describe('A Type Mapping Repository', async () => {
  expect(mappings.value[0].transformations?.length).eq(1);
 
  // next export the mappings into a separate container
- exported = await repo.importToDataSource(dataSource2ID, user, mapping);
+ exported = await repo.importToDataSource(dataSource2ID, user, active, mapping);
  for (const result of exported) {
  expect(result.isError).false;
  expect(result.value.id).not.eq(mapping.id); // should be a new mapping