Extract cyhy orgs & vuln scans data

backend/package.json

@@ -25,6 +25,7 @@
  "helmet": "^4.1.1",
  "http-proxy-middleware": "^2.0.6",
  "ip": "^1.1.9",
+ "ip-address": "^9.0.5",
  "jsdom": "^22.1",
  "jsonwebtoken": "^9.0.2",
  "jwks-rsa": "^3.0",
@@ -39,6 +40,7 @@
  "reflect-metadata": "^0.1.13",
  "serverless-http": "^3.2.0",
  "ssl-checker": "^2.0.7",
+ "tar": "^7.0.0",
  "tough-cookie": "^4.1.3",
  "typeorm": "^0.2.45",
  "utf-8-validate": "^6.0.3",

backend/src/api/scans.ts

@@ -88,6 +88,14 @@ export const SCAN_SCHEMA: ScanSchema = {
  description:
  "Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog."
  },
+ vulnScanningSync: {
+ type: 'fargate',
+ isPassive: true,
+ global: true,
+ description: 'Pull in vulnerability data from VSs Vulnerability database',
+ cpu: '1024',
+ memory: '8192'
+ },
  cveSync: {
  type: 'fargate',
  isPassive: true,

backend/src/models/connection.ts

@@ -37,7 +37,6 @@ import {
  PortScan,
  PrecertScan,
  Report,
- Request,
  Sector,
  Snapshot,
  SslyzeScan,
@@ -54,6 +53,14 @@ let connection: Connection | null = null;
 let dl_connection: Connection | null = null;
 
 const connectDl = async (logging?: boolean) => {
+ // process.env.DB_HOST = 'db';
+ // process.env.MDL_USERNAME = 'mdl';
+ // process.env.MDL_PASSWORD = 'password';
+ // process.env.MDL_NAME = 'crossfeed_mini_datalake';
+
+ // console.log(process.env.MDL_USERNAME)
+ // console.log(process.env.MDL_PASSWORD)
+ // console.log(process.env.MDL_NAME)
  const dl_connection = createConnection({
  type: 'postgres',
  host: process.env.DB_HOST,
@@ -77,7 +84,6 @@ const connectDl = async (logging?: boolean) => {
  PortScan,
  PrecertScan,
  Report,
- Request,
  Sector,
  Snapshot,
  SslyzeScan,
@@ -89,7 +95,7 @@ const connectDl = async (logging?: boolean) => {
  VulnScan
  ],
  synchronize: false,
- name: 'mini_data_lake',
+ name: 'default',
  dropSchema: false,
  logging: logging ?? false,
  cache: true
@@ -99,6 +105,7 @@ const connectDl = async (logging?: boolean) => {
 
 export const connectToDatalake = async (logging?: boolean) => {
  if (!dl_connection?.isConnected) {
+ console.log('Connected to datalake');
  dl_connection = await connectDl(logging);
  } else {
  console.log("didn't connect");

backend/src/models/index.ts

@@ -34,7 +34,6 @@ export * from './mini_data_lake/locations';
 export * from './mini_data_lake/port_scans';
 export * from './mini_data_lake/precert_scans';
 export * from './mini_data_lake/reports';
-export * from './mini_data_lake/requests';
 export * from './mini_data_lake/sectors';
 export * from './mini_data_lake/snapshots';
 export * from './mini_data_lake/sslyze_scan';

backend/src/models/mini_data_lake/cidrs.ts

@@ -6,10 +6,10 @@ import {
  CreateDateColumn,
  BaseEntity,
  ManyToMany,
- JoinTable
+ JoinTable,
+ UpdateDateColumn
 } from 'typeorm';
 
-import { Request } from './requests';
 import { Organization } from './organizations';
 @Entity()
 export class Cidr extends BaseEntity {
@@ -22,7 +22,7 @@ export class Cidr extends BaseEntity {
  @Index()
  @Column({
  nullable: true,
- type: 'cidr',
+ type: 'inet',
  unique: true
  })
  network: string | null;
@@ -42,12 +42,8 @@ export class Cidr extends BaseEntity {
  @Column({ nullable: true })
  retired: boolean;
 
- @ManyToMany((type) => Request, (request) => request.cidrs, {
- onDelete: 'CASCADE',
- onUpdate: 'CASCADE'
- })
- @JoinTable()
- requests: Request[];
+ @UpdateDateColumn()
+ updatedAt: Date | null;
 
  @ManyToMany((type) => Organization, (org) => org.cidrs, {
  onDelete: 'CASCADE',

backend/src/models/mini_data_lake/ips.ts

@@ -3,11 +3,14 @@
 
 import {
  Entity,
+ Unique,
  Column,
  PrimaryGeneratedColumn,
  BaseEntity,
  OneToMany,
- ManyToOne
+ ManyToOne,
+ UpdateDateColumn,
+ CreateDateColumn
 } from 'typeorm';
 import { Domain } from './domains';
 import { HostScan } from './host_scans';
@@ -17,6 +20,7 @@ import { Ticket } from './tickets';
 import { VulnScan } from './vuln_scans';
 import { PortScan } from './port_scans';
 @Entity()
+@Unique(['ip', 'organization'])
 export class Ip extends BaseEntity {
  @PrimaryGeneratedColumn('uuid')
  id: string;
@@ -28,8 +32,11 @@ export class Ip extends BaseEntity {
  })
  organization: Organization;
 
- @Column({ nullable: true, type: 'timestamp' })
- createdTimestamp: Date | null;
+ @CreateDateColumn()
+ createdTimestamp: Date;
+
+ @UpdateDateColumn()
+ updatedTimestamp: Date | null;
 
  @Column({ nullable: true, type: 'timestamp' })
  lastSeenTimestamp: Date | null;

backend/src/models/mini_data_lake/kevs.ts

@@ -26,6 +26,50 @@ export class Kev extends BaseEntity {
  @Column({ nullable: true })
  knownRansomware: boolean;
 
+ @Column({
+ nullable: true,
+ type: 'varchar'
+ })
+ vendorProject: string | null;
+ @Column({
+ nullable: true,
+ type: 'varchar'
+ })
+ product: string | null;
+
+ @Column({
+ nullable: true,
+ type: 'varchar'
+ })
+ vulnerabilityName: string | null;
+
+ @Column({
+ nullable: true,
+ type: 'timestamp'
+ })
+ dateAdded: Date | null;
+
+ @Column({
+ nullable: true,
+ type: 'varchar'
+ })
+ shortDescription: string | null;
+
+ @Column({
+ nullable: true,
+ type: 'varchar'
+ })
+ requiredAction: string | null;
+
+ @Column({ nullable: true, type: 'timestamp' })
+ dueDate: Date | null;
+
+ @Column({
+ nullable: true,
+ type: 'varchar'
+ })
+ notes: string | null;
+
  @OneToMany((type) => Ticket, (ticket) => ticket.kev, {
  onDelete: 'CASCADE',
  onUpdate: 'CASCADE'

backend/src/models/mini_data_lake/locations.ts

@@ -7,8 +7,7 @@ import {
  Column,
  PrimaryGeneratedColumn,
  BaseEntity,
- ManyToMany,
- JoinTable
+ OneToMany
 } from 'typeorm';
 
 import { Organization } from './organizations';
@@ -27,7 +26,7 @@ export class Location extends BaseEntity {
  nullable: true,
  type: 'varchar'
  })
- countryAbvr: string | null;
+ countryAbrv: string | null;
 
  @Column({
  nullable: true,
@@ -72,10 +71,9 @@ export class Location extends BaseEntity {
  })
  state: string | null;
 
- @ManyToMany((type) => Organization, (org) => org.locations, {
+ @OneToMany((type) => Organization, (org) => org.location, {
  onDelete: 'CASCADE',
  onUpdate: 'CASCADE'
  })
- @JoinTable()
- organizations: Organization[];
+ organizations?: Organization[];
 }

backend/src/models/mini_data_lake/organizations.ts

@@ -9,7 +9,9 @@ import {
  BaseEntity,
  OneToMany,
  ManyToMany,
- ManyToOne
+ ManyToOne,
+ CreateDateColumn,
+ UpdateDateColumn
 } from 'typeorm';
 import { Domain } from './domains';
 import { Ip } from './ips';
@@ -18,7 +20,6 @@ import { Contact } from './contacts';
 import { Tag } from './tag';
 import { Sector } from './sectors';
 import { Report } from './reports';
-import { Request } from './requests';
 import { SslyzeScan } from './sslyze_scan';
 import { Snapshot } from './snapshots';
 import { Tally } from './tallies';
@@ -48,26 +49,17 @@ export class Organization extends BaseEntity {
  acronym: string | null;
 
  @Column({ nullable: true, type: 'timestamp' })
- firstEngageDate: Date | null;
+ enrolledInVsTimestamp: Date | null;
 
  @Column({ nullable: true, type: 'timestamp' })
- createdDate: Date | null;
+ periodStartVsTimestamp: Date | null;
 
- @Column({
- nullable: true,
- type: 'varchar'
- })
- createdEmplyeeId: string | null;
+ @CreateDateColumn()
+ createdDate: Date;
 
- @Column({ nullable: true, type: 'timestamp' })
+ @UpdateDateColumn()
  updatedDate: Date | null;
 
- @Column({
- nullable: true,
- type: 'varchar'
- })
- updatedEmployeeId: string | null;
-
  @Column({ nullable: true })
  retired: boolean;
 
@@ -92,6 +84,44 @@ export class Organization extends BaseEntity {
  @Column({ nullable: true })
  stakeholder: boolean;
 
+ @Column({
+ nullable: true,
+ type: 'varchar'
+ })
+ reportPeriod: string | null;
+
+ @Column({
+ nullable: true,
+ type: 'varchar'
+ })
+ initStage: string | null;
+
+ @Column({
+ nullable: true,
+ type: 'varchar'
+ })
+ scheduler: string | null;
+
+ @Column('varchar', { array: true, default: [], nullable: true })
+ reportTypes: string[] | null;
+
+ @Column('varchar', { array: true, default: [], nullable: true })
+ scanTypes: string[] | null;
+
+ @Column({
+ nullable: true,
+ type: 'jsonb',
+ default: []
+ })
+ scanWindows: Object[] | null;
+
+ @Column({
+ nullable: true,
+ type: 'jsonb',
+ default: []
+ })
+ scanLimits: Object[] | null;
+
  @OneToMany((type) => Domain, (domain) => domain.organization, {
  onDelete: 'CASCADE',
  onUpdate: 'CASCADE'
@@ -110,11 +140,11 @@ export class Organization extends BaseEntity {
  })
  tickets: Ticket[];
 
- @ManyToMany((type) => Location, (location) => location.organizations, {
+ @ManyToOne((type) => Location, (location) => location.organizations, {
  onDelete: 'CASCADE',
- onUpdate: 'CASCADE'
+ nullable: true
  })
- locations: Location[];
+ location: Location;
 
  @ManyToMany((type) => Contact, (contact) => contact.organizations, {
  onDelete: 'CASCADE',
@@ -158,12 +188,6 @@ export class Organization extends BaseEntity {
  })
  reports: Report[];
 
- @OneToMany((type) => Request, (request) => request.organization, {
- onDelete: 'CASCADE',
- onUpdate: 'CASCADE'
- })
- requests: Request[];
-
  @OneToMany((type) => SslyzeScan, (sslyze) => sslyze.organization, {
  onDelete: 'CASCADE',
  onUpdate: 'CASCADE'
@@ -190,11 +214,6 @@ export class Organization extends BaseEntity {
  onUpdate: 'CASCADE'
  }
  )
- @Column({
- nullable: true,
- type: 'varchar'
- })
- reportPeriod: string | null;
  trustymailScans: TrustymailScan[];
 
  @OneToMany((type) => VulnScan, (vuln_scan) => vuln_scan.organization, {