Merge pull request #884 from JamesHabben/lava-output

minor module updates and testing update

admin/test/cases/testdata.accountConfig.json

@@ -0,0 +1,89 @@
+{
+  "josh_ios15_ffs": {
+    "description": "",
+    "maker": "",
+    "make_data": {
+      "input_data_path": "/Users/jameshabben/Documents/phone-images/Josh/iOS_15_Public_Image.tar.gz",
+      "os": "macOS-15.0-x86_64-i386-64bit",
+      "timestamp": "2024-10-17T13:27:09.586077",
+      "last_commit": {
+        "hash": "809b08c7dac89b2f2bcc7d6692b9981e3cf39913",
+        "author_name": "Johann Polewczyk",
+        "author_email": "johannplw@me.com",
+        "date": "2024-10-17T19:06:04+02:00",
+        "message": "resolve kml error issue with 'all' in output_types"
+      }
+    },
+    "artifacts": {
+      "accountConfig": {
+        "search_patterns": [
+          "*/com.apple.accounts.exists.plist"
+        ],
+        "file_count": 1,
+        "expected_output": {
+          "headers": [],
+          "data": []
+        }
+      }
+    },
+    "image_name": "josh_ios15_ffs"
+  },
+  "mvs_ios_2023": {
+    "description": "",
+    "maker": "",
+    "make_data": {
+      "input_data_path": "/Users/jameshabben/Documents/phone-images/magnet/00008101-0010541A1130001E_files_full-001.zip",
+      "os": "macOS-15.0-x86_64-i386-64bit",
+      "timestamp": "2024-10-17T13:44:38.312743",
+      "last_commit": {
+        "hash": "809b08c7dac89b2f2bcc7d6692b9981e3cf39913",
+        "author_name": "Johann Polewczyk",
+        "author_email": "johannplw@me.com",
+        "date": "2024-10-17T19:06:04+02:00",
+        "message": "resolve kml error issue with 'all' in output_types"
+      }
+    },
+    "artifacts": {
+      "accountConfig": {
+        "search_patterns": [
+          "*/com.apple.accounts.exists.plist"
+        ],
+        "file_count": 1,
+        "expected_output": {
+          "headers": [],
+          "data": []
+        }
+      }
+    },
+    "image_name": "mvs_ios_2023"
+  },
+  "belkasoft_ctf6_ios_device1": {
+    "description": "",
+    "maker": "",
+    "make_data": {
+      "input_data_path": "/Users/jameshabben/Documents/phone-images/belkasoft/BelkaCTF_6_CASE240405_D201AP.tar",
+      "os": "macOS-15.0-x86_64-i386-64bit",
+      "timestamp": "2024-10-17T13:51:51.068948",
+      "last_commit": {
+        "hash": "809b08c7dac89b2f2bcc7d6692b9981e3cf39913",
+        "author_name": "Johann Polewczyk",
+        "author_email": "johannplw@me.com",
+        "date": "2024-10-17T19:06:04+02:00",
+        "message": "resolve kml error issue with 'all' in output_types"
+      }
+    },
+    "artifacts": {
+      "accountConfig": {
+        "search_patterns": [
+          "*/com.apple.accounts.exists.plist"
+        ],
+        "file_count": 1,
+        "expected_output": {
+          "headers": [],
+          "data": []
+        }
+      }
+    },
+    "image_name": "belkasoft_ctf6_ios_device1"
+  }
+}

admin/test/cases/testdata.accountData.json

@@ -0,0 +1,89 @@
+{
+  "josh_ios15_ffs": {
+    "description": "",
+    "maker": "",
+    "make_data": {
+      "input_data_path": "/Users/jameshabben/Documents/phone-images/Josh/iOS_15_Public_Image.tar.gz",
+      "os": "macOS-15.0-x86_64-i386-64bit",
+      "timestamp": "2024-10-17T12:09:27.445682",
+      "last_commit": {
+        "hash": "809b08c7dac89b2f2bcc7d6692b9981e3cf39913",
+        "author_name": "Johann Polewczyk",
+        "author_email": "johannplw@me.com",
+        "date": "2024-10-17T19:06:04+02:00",
+        "message": "resolve kml error issue with 'all' in output_types"
+      }
+    },
+    "artifacts": {
+      "accountData": {
+        "search_patterns": [
+          "*/mobile/Library/Accounts/Accounts3.sqlite*"
+        ],
+        "file_count": 3,
+        "expected_output": {
+          "headers": [],
+          "data": []
+        }
+      }
+    },
+    "image_name": "josh_ios15_ffs"
+  },
+  "mvs_ios_2023": {
+    "description": "",
+    "maker": "",
+    "make_data": {
+      "input_data_path": "/Users/jameshabben/Documents/phone-images/magnet/00008101-0010541A1130001E_files_full-001.zip",
+      "os": "macOS-15.0-x86_64-i386-64bit",
+      "timestamp": "2024-10-17T13:25:57.671900",
+      "last_commit": {
+        "hash": "809b08c7dac89b2f2bcc7d6692b9981e3cf39913",
+        "author_name": "Johann Polewczyk",
+        "author_email": "johannplw@me.com",
+        "date": "2024-10-17T19:06:04+02:00",
+        "message": "resolve kml error issue with 'all' in output_types"
+      }
+    },
+    "artifacts": {
+      "accountData": {
+        "search_patterns": [
+          "*/mobile/Library/Accounts/Accounts3.sqlite*"
+        ],
+        "file_count": 3,
+        "expected_output": {
+          "headers": [],
+          "data": []
+        }
+      }
+    },
+    "image_name": "mvs_ios_2023"
+  },
+  "belkasoft_ctf6_ios_device1": {
+    "description": "",
+    "maker": "",
+    "make_data": {
+      "input_data_path": "/Users/jameshabben/Documents/phone-images/belkasoft/BelkaCTF_6_CASE240405_D201AP.tar",
+      "os": "macOS-15.0-x86_64-i386-64bit",
+      "timestamp": "2024-10-17T13:26:09.085469",
+      "last_commit": {
+        "hash": "809b08c7dac89b2f2bcc7d6692b9981e3cf39913",
+        "author_name": "Johann Polewczyk",
+        "author_email": "johannplw@me.com",
+        "date": "2024-10-17T19:06:04+02:00",
+        "message": "resolve kml error issue with 'all' in output_types"
+      }
+    },
+    "artifacts": {
+      "accountData": {
+        "search_patterns": [
+          "*/mobile/Library/Accounts/Accounts3.sqlite*"
+        ],
+        "file_count": 3,
+        "expected_output": {
+          "headers": [],
+          "data": []
+        }
+      }
+    },
+    "image_name": "belkasoft_ctf6_ios_device1"
+  }
+}

admin/test/scripts/make_test_data.py

@@ -14,6 +14,7 @@
 import csv
 from io import StringIO
 import textwrap
+import glob
 
 # Add the correct path to the system path
 repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
@@ -81,6 +82,9 @@ def read_filepath_list(file_path_list):
 
 def match_files_from_list(filepath_list, patterns):
     matching_files = []
+    # Ensure patterns is always a tuple or list
+    if isinstance(patterns, str):
+        patterns = (patterns,)
     for filepath in filepath_list:
         if any(fnmatch.fnmatch(filepath, pattern) for pattern in patterns):
             matching_files.append(filepath)
@@ -336,11 +340,15 @@ def create_test_data(module_name, image_name=None, case_number=None, input_file=
     print(f"\nTotal processing time: {overall_end_time - overall_start_time:.2f} seconds")
     print(f"Script completed at: {local_overall_end_time}")
 
-def prompt_for_image():
+def prompt_for_image(module_name):
     manifest = load_image_manifest()
     print("\nAvailable images:")
     for i, image in enumerate(manifest, 1):
-        print(f"{i}. {image['image_name']}")
+        image_name = image['image_name']
+        has_case_data = check_existing_case_data(module_name, image_name)
+        case_data_status = "✓ Has case data" if has_case_data else "✗ No case data"
+
+        print(f"{i}. {image_name} [{case_data_status}]")
         if 'description' in image:
             wrapped_description = textwrap.wrap(image['description'], width=70, initial_indent='   ', subsequent_indent='   ')
             print('\n'.join(wrapped_description))
@@ -356,6 +364,12 @@ def prompt_for_image():
         except ValueError:
             print("Invalid input. Please enter a number.")
 
+def check_existing_case_data(module_name, image_name):
+    cases_dir = os.path.join(repo_root, 'admin', 'test', 'cases', 'data')
+    pattern = f"testdata.{module_name}.*.{image_name}.zip"
+    existing_files = glob.glob(os.path.join(cases_dir, pattern))
+    return len(existing_files) > 0
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Create test data for artifacts")
     parser.add_argument("module_name", help="Name of the module (e.g., keyboard or keyboard.py)")
@@ -375,14 +389,23 @@ def prompt_for_image():
     print(f"Script started at: {script_start_time}")
 
     if args.image_prompt:
-        selected_image = prompt_for_image()
+        selected_image = prompt_for_image(module_name)
         args.image = selected_image
 
     if args.image:
         try:
             image_info = get_image_info(args.image)
             print(f"Using image: {args.image}")
             print(f"Image path: {image_info['input_file']}")
+
+            # Check if case data already exists
+            if check_existing_case_data(module_name, args.image):
+                print(f"Note: Case data already exists for module '{module_name}' using image '{args.image}'.")
+                proceed = input("Do you want to proceed and potentially overwrite existing data? (y/n): ")
+                if proceed.lower() != 'y':
+                    print("Operation aborted.")
+                    sys.exit(0)
+
             create_test_data(module_name, image_name=args.image, input_file=image_info['input_file'])
         except (ValueError, FileNotFoundError) as e:
             print(f"Error: {e}")

scripts/artifacts/accountConfig.py

@@ -1,5 +1,5 @@
 __artifacts_v2__ = {
-    "get_confaccts": {
+    "accountConfig": {
         "name": "Account Configuration",
         "description": "Extracts account configuration information",
         "author": "@AlexisBrignoni",
@@ -17,7 +17,7 @@
 from scripts.ilapfuncs import artifact_processor
 
 @artifact_processor
-def get_confaccts(files_found, report_folder, seeker, wrap_text, timezone_offset):
+def accountConfig(files_found, report_folder, seeker, wrap_text, timezone_offset):
     data_list = []
     data_headers = ()
     source_path = str(files_found[0])

scripts/artifacts/accountData.py

@@ -1,5 +1,5 @@
 __artifacts_v2__ = {
-    "get_accs": {
+    "accountData": {
         "name": "Account Data",
         "description": "Extract information about configured user accounts",
         "author": "@AlexisBrignoni",
@@ -8,7 +8,7 @@
         "requirements": "none",
         "category": "Accounts",
         "notes": "",
-        "paths": ('*/mobile/Library/Accounts/Accounts3.sqlite*'),
+        "paths": ('*/mobile/Library/Accounts/Accounts3.sqlite*',),
         "output_types": "standard"
     }
 }
@@ -17,7 +17,7 @@
 from scripts.ilapfuncs import artifact_processor, open_sqlite_db_readonly, convert_ts_human_to_timezone_offset
 
 @artifact_processor
-def get_accs(files_found, report_folder, seeker, wrap_text, timezone_offset):
+def accountData(files_found, report_folder, seeker, wrap_text, timezone_offset):
     data_list = []
     data_headers = ()
     source_path = ''

scripts/artifacts/ATXDatastore.py → scripts/artifacts/atxDatastore.py

@@ -1,5 +1,5 @@
 __artifacts_v2__ = {
-    "get_atxDatastore": {
+    "atxDatastore": {
         "name": "iOS ATXDatastore",
         "description": "Parses ATXDataStore and matches actions with Frequent locations, when available.",
         "author": "@magpol",
@@ -16,7 +16,7 @@
 from scripts.ilapfuncs import artifact_processor, open_sqlite_db_readonly, convert_ts_human_to_timezone_offset 
 
 @artifact_processor
-def get_atxDatastore(files_found, report_folder, seeker, wrap_text, timezone_offset):
+def atxDatastore(files_found, report_folder, seeker, wrap_text, timezone_offset):
     data_list = []
     data_headers = ()
     source_path = ''