From ca2e71f592b147b990677691101fc785cfdb9c75 Mon Sep 17 00:00:00 2001 From: schmelz21 <81192860+schmelz21@users.noreply.github.com> Date: Tue, 1 Oct 2024 13:43:26 -0400 Subject: [PATCH] Revert "Only display domains that are directly attributable to a cidr" --- backend/src/api/domains.ts | 4 -- backend/src/api/scans.ts | 7 --- backend/src/api/stats.ts | 4 -- backend/src/api/vulnerabilities.ts | 4 -- backend/src/models/domain.ts | 10 ---- backend/src/tasks/flagFloatingIps.ts | 28 ---------- backend/src/tasks/helpers/checkIpInCidr.ts | 61 ---------------------- backend/src/tasks/search-sync-domains.ts | 4 -- backend/src/worker.ts | 2 - 9 files changed, 124 deletions(-) delete mode 100644 backend/src/tasks/flagFloatingIps.ts delete mode 100644 backend/src/tasks/helpers/checkIpInCidr.ts diff --git a/backend/src/api/domains.ts b/backend/src/api/domains.ts index 2e651a82..dd8924e5 100644 --- a/backend/src/api/domains.ts +++ b/backend/src/api/domains.ts @@ -154,10 +154,6 @@ class DomainSearch { }); } - qs.andWhere( - 'domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true)' - ); - await this.filterResultQueryset(qs, event); return qs.getManyAndCount(); } diff --git a/backend/src/api/scans.ts b/backend/src/api/scans.ts index 2e6155dd..79d20483 100644 --- a/backend/src/api/scans.ts +++ b/backend/src/api/scans.ts @@ -128,13 +128,6 @@ export const SCAN_SCHEMA: ScanSchema = { description: 'Open source tool that integrates passive APIs in order to discover target subdomains' }, - flagFloatingIps: { - type: 'fargate', - isPassive: true, - global: true, - description: - 'Loops through all domains and determines if their associated IP can be found in a report Cidr block.' - }, hibp: { type: 'fargate', isPassive: true, diff --git a/backend/src/api/stats.ts b/backend/src/api/stats.ts index d9523a20..54238947 100644 --- a/backend/src/api/stats.ts +++ b/backend/src/api/stats.ts @@ -94,10 +94,6 @@ export const get = wrapHandler(async (event) => { }); } - qs.andWhere( - 'domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true)' - ); - // Handles the case where no orgs and no regions are set, and we pull stats for a region that will never exist if ( search.filters?.organizations?.length === 0 && diff --git a/backend/src/api/vulnerabilities.ts b/backend/src/api/vulnerabilities.ts index 4897b258..65b71ad0 100644 --- a/backend/src/api/vulnerabilities.ts +++ b/backend/src/api/vulnerabilities.ts @@ -173,10 +173,6 @@ class VulnerabilitySearch { .leftJoinAndSelect('domain.organization', 'organization') .leftJoinAndSelect('vulnerability.service', 'service'); - qs.andWhere( - 'domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true)' - ); - if (groupBy) { qs = qs .groupBy('title, cve, "isKev", description, severity') diff --git a/backend/src/models/domain.ts b/backend/src/models/domain.ts index c33d9d10..2a6be83c 100644 --- a/backend/src/models/domain.ts +++ b/backend/src/models/domain.ts @@ -111,16 +111,6 @@ export class Domain extends BaseEntity { }) cloudHosted: boolean; - @Column({ - default: false - }) - fromCidr: boolean; - - @Column({ - default: false - }) - isFceb: boolean; - /** SSL Certificate information */ @Column({ type: 'jsonb', diff --git a/backend/src/tasks/flagFloatingIps.ts b/backend/src/tasks/flagFloatingIps.ts deleted file mode 100644 index e09b655d..00000000 --- a/backend/src/tasks/flagFloatingIps.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { CommandOptions } from './ecs-client'; -import checkIpInCidr from './helpers/checkIpInCidr'; -import { Organization, connectToDatabase } from '../models'; - -export const handler = async (commandOptions: CommandOptions) => { - const db_connection = await connectToDatabase(); - const organization_repo = db_connection.getRepository(Organization); - const organizations = await organization_repo.find({ - relations: ['domains'] - }); - for (const organization of organizations) { - for (const domain of organization.domains) { - if (domain.ip) { - const cidrSectorDict = await checkIpInCidr( - domain.ip, - organization.acronym - ); - if (cidrSectorDict['isInCidr']) { - domain.fromCidr = true; - } - if (cidrSectorDict['isExecutive']) { - domain.isFceb = true; - } - domain.save(); - } - } - } -}; \ No newline at end of file diff --git a/backend/src/tasks/helpers/checkIpInCidr.ts b/backend/src/tasks/helpers/checkIpInCidr.ts deleted file mode 100644 index 2101abb3..00000000 --- a/backend/src/tasks/helpers/checkIpInCidr.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { getRepository } from 'typeorm'; -import { Cidr, DL_Organization, connectToDatalake2 } from '../../models'; - -export default async ( - ip: string, - acronym: string -): Promise<{ isInCidr: boolean; isExecutive: boolean }> => { - // await connectToDatalake2() - // const cidrRepository = getRepository(Cidr); - // const organizationRepository = getRepository(DL_Organization); - - // Find the organization by acronym - const mdl_connection = await connectToDatalake2(); - const mdl_organization_repo = mdl_connection.getRepository(DL_Organization); - const organization = await mdl_organization_repo.findOne({ - where: { acronym }, - relations: ['cidrs', 'sectors', 'parent'] - }); - - if (!organization) { - return { isInCidr: false, isExecutive: false }; - } - - const isOrganizationExecutive = async ( - org: DL_Organization - ): Promise => { - if (org.sectors.some((sector) => sector.acronym === 'EXECUTIVE')) { - return true; - } - if (org.parent) { - const parentOrg = await mdl_organization_repo.findOne({ - where: { id: org.parent.id }, - relations: ['sectors'] - }); - - return parentOrg ? await isOrganizationExecutive(parentOrg) : false; - } - return false; - }; - - const isExecutive = await isOrganizationExecutive(organization); - - // Get CIDRs related to the organization - const cidrs = organization.cidrs.map((cidr) => cidr.network); - - if (cidrs.length === 0) { - return { isInCidr: false, isExecutive }; // No CIDRs associated with the organization - } - - // Check if the IP is in any of the CIDRs - const mdl_cidr_repo = mdl_connection.getRepository(Cidr); - const result = await mdl_cidr_repo - .createQueryBuilder('cidr') - .where('cidr.network >>= :ip', { ip }) - .andWhere('cidr.id IN (:...cidrIds)', { - cidrIds: organization.cidrs.map((cidr) => cidr.id) - }) - .getCount(); - - return { isInCidr: result > 0, isExecutive }; -}; diff --git a/backend/src/tasks/search-sync-domains.ts b/backend/src/tasks/search-sync-domains.ts index fb870e99..ac9337a6 100644 --- a/backend/src/tasks/search-sync-domains.ts +++ b/backend/src/tasks/search-sync-domains.ts @@ -40,10 +40,6 @@ export const handler = async (commandOptions: CommandOptions) => { qs.where('organization.id=:org', { org: organizationId }); } - qs.andWhere( - 'domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true)' - ); - const domainIds = (await qs.getMany()).map((e) => e.id); console.log(`Got ${domainIds.length} domains.`); if (domainIds.length) { diff --git a/backend/src/worker.ts b/backend/src/worker.ts index dd3f3901..94f0ca6e 100644 --- a/backend/src/worker.ts +++ b/backend/src/worker.ts @@ -23,7 +23,6 @@ import { handler as sslyze } from './tasks/sslyze'; import { handler as trustymail } from './tasks/trustymail'; import { handler as vulnSync } from './tasks/vuln-sync'; import { handler as vulnScanningSync } from './tasks/vs_sync'; -import { handler as flagFloatingIps } from './tasks/flagFloatingIps'; import { handler as xpanseSync } from './tasks/xpanse-sync'; import { SCAN_SCHEMA } from './api/scans'; @@ -48,7 +47,6 @@ async function main() { dnstwist, dotgov, findomain, - flagFloatingIps, intrigueIdent, lookingGlass, portscanner,