feat/add include_slide_notes parameter (#455)

## Description
* Added `include_slide_notes` parameter, default is `True`. Works for
`.ppt` and `.pptx` file extensions.
* Added two new files in `sample-docs`: `sample-docs/notes.ppt`,
`sample-docs/notes.pptx` that include notes on their slides. This is to
easily test the functionality, as there are no existing PowerPoint files
that include slide notes.

## Testing
```
#  using default value (True) returns additional NarrativeText element that contains notes
curl -X 'POST'   'http://localhost:8000/general/v0/general'   -H 'accept: application/json'   -H 'Content-Type: multipart/form-data'   -F 'files=@sample-docs/notes.pptx'   -F 'output_format="text/csv"' 

# explicit include_slide_notes=True returns additional NarrativeText element that contains notes
curl -X 'POST'   'http://localhost:8000/general/v0/general'   -H 'accept: application/json'   -H 'Content-Type: multipart/form-data'   -F 'files=@sample-docs/notes.pptx'   -F 'output_format="text/csv"' -F 'include_slide_notes=True'

# explicit include_slide_notes=False returns no NarrativeText element 
curl -X 'POST'   'http://localhost:8000/general/v0/general'   -H 'accept: application/json'   -H 'Content-Type: multipart/form-data'   -F 'files=@sample-docs/notes.pptx'   -F 'output_format="text/csv"' -F 'include_slide_notes=False'
```

Same with file `notes.ppt`

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.0.80-dev0
+
+* Add `include_slide_notes` parameter, indicating whether slide notes in `ppt` and `pptx` files should be partitioned. Default is `True`. Now, when slide notes are present in the file, they will be included alongside other elements, which may shift the index numbers of non-note elements. 
+
 ## 0.0.79
 
 * Bump to `unstructured` 0.15.7

prepline_general/api/app.py

@@ -13,7 +13,7 @@
 app = FastAPI(
  title="Unstructured Pipeline API",
  summary="Partition documents with the Unstructured library",
- version="0.0.79",
+ version="0.0.80",
  docs_url="/general/docs",
  openapi_url="/general/openapi.json",
  servers=[

prepline_general/api/general.py

@@ -273,6 +273,7 @@ def pipeline_api(
  extract_image_block_types: Optional[List[str]] = None,
  unique_element_ids: Optional[bool] = False,
  starting_page_number: Optional[int] = None,
+ include_slide_notes: Optional[bool] = True,
 ) -> List[Dict[str, Any]] | str:
  if filename.endswith(".msg"):
  # Note(yuming): convert file type for msg files
@@ -316,6 +317,7 @@ def pipeline_api(
  "overlap": overlap,
  "overlap_all": overlap_all,
  "starting_page_number": starting_page_number,
+ "include_slide_notes": include_slide_notes,
  },
  default=str,
  )
@@ -373,6 +375,7 @@ def pipeline_api(
  "extract_image_block_types": extract_image_block_types,
  "extract_image_block_to_payload": extract_image_block_to_payload,
  "unique_element_ids": unique_element_ids,
+ "include_slide_notes": include_slide_notes,
  },
  default=str,
  )
@@ -403,6 +406,7 @@ def pipeline_api(
  "extract_image_block_to_payload": extract_image_block_to_payload,
  "unique_element_ids": unique_element_ids,
  "starting_page_number": starting_page_number,
+ "include_slide_notes": include_slide_notes,
  }
 
  if file_content_type == "application/pdf" and pdf_parallel_mode_enabled:
@@ -649,7 +653,7 @@ def return_content_type(filename: str):
 
 
 @router.get("/general/v0/general", include_in_schema=False)
-@router.get("/general/v0.0.79/general", include_in_schema=False)
+@router.get("/general/v0.0.80/general", include_in_schema=False)
 async def handle_invalid_get_request():
  raise HTTPException(
  status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail="Only POST requests are supported."
@@ -664,7 +668,7 @@ async def handle_invalid_get_request():
  description="Description",
  operation_id="partition_parameters",
 )
-@router.post("/general/v0.0.79/general", include_in_schema=False)
+@router.post("/general/v0.0.80/general", include_in_schema=False)
 def general_partition(
  request: Request,
  # cannot use annotated type here because of a bug described here:
@@ -747,6 +751,7 @@ def response_generator(is_multipart: bool):
  overlap=form_params.overlap,
  overlap_all=form_params.overlap_all,
  starting_page_number=form_params.starting_page_number,
+ include_slide_notes=form_params.include_slide_notes,
  )
 
  yield (

prepline_general/api/models/form_params.py

@@ -36,6 +36,7 @@ class GeneralFormParams(BaseModel):
  overlap: int
  overlap_all: bool
  starting_page_number: Optional[int] = None
+ include_slide_notes: bool
 
  @classmethod
  def as_form(
@@ -246,6 +247,17 @@ def as_form(
  example=3,
  ),
  ] = None,
+ include_slide_notes: Annotated[
+ bool,
+ Form(
+ title="include_slide_notes",
+ description=(
+ "When `True`, slide notes from .ppt and .pptx files"
+ " will be included in the response. Default: `True`"
+ ),
+ example=False,
+ ),
+ ] = True,
  ) -> "GeneralFormParams":
  return cls(
  xml_keep_tags=xml_keep_tags,
@@ -273,4 +285,5 @@ def as_form(
  overlap_all=overlap_all,
  unique_element_ids=unique_element_ids,
  starting_page_number=starting_page_number,
+ include_slide_notes=include_slide_notes,
  )

preprocessing-pipeline-family.yaml

@@ -1,2 +1,2 @@
 name: general
-version: 0.0.79
+version: 0.0.80

test_general/api/test_app.py

@@ -701,6 +701,7 @@ def test_parallel_mode_passes_params(monkeypatch):
  "new_after_n_chars": "1501",
  "overlap": "25",
  "overlap_all": "true",
+ "include_slide_notes": "true",
  },
  )
 
@@ -733,6 +734,7 @@ def test_parallel_mode_passes_params(monkeypatch):
  new_after_n_chars=1501,
  overlap=25,
  overlap_all=True,
+ include_slide_notes=True,
  )
 
 
@@ -1147,3 +1149,38 @@ def test__set_pdf_infer_table_structure(
  )
  is expected
  )
+
+
+@pytest.mark.parametrize(
+ ("test_default", "include_slide_notes", "test_file"),
+ [
+ (True, None, Path("sample-docs") / "notes.ppt"),
+ (True, None, Path("sample-docs") / "notes.pptx"),
+ (False, True, Path("sample-docs") / "notes.ppt"),
+ (False, True, Path("sample-docs") / "notes.pptx"),
+ (False, False, Path("sample-docs") / "notes.ppt"),
+ (False, False, Path("sample-docs") / "notes.pptx"),
+ ],
+)
+def test_include_slide_notes(monkeypatch, test_default, include_slide_notes, test_file):
+ """
+ Verifies that the output includes slide notes when the include_slide_notes parameter
+ is left as default or explicitly set to True.
+ """
+ client = TestClient(app)
+ data = (
+ {"output_format": "text/csv"}
+ if test_default
+ else {"include_slide_notes": str(include_slide_notes), "output_format": "text/csv"}
+ )
+ response = client.post(
+ MAIN_API_ROUTE,
+ files=[("files", (str(test_file), open(test_file, "rb")))],
+ data=data,
+ )
+ df = pd.read_csv(io.StringIO(response.text))
+
+ if include_slide_notes or test_default:
+ assert "Here are important notes" == df["text"][0]
+ else:
+ assert "Here are important notes" != df["text"][0]