Skip to content

Commit

Permalink
Update parquet executor spec (#31)
Browse files Browse the repository at this point in the history
  • Loading branch information
TomPham97 authored Nov 12, 2023
1 parent bb4437f commit 2b9fedb
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions _posts/2020-09-13-tfx-data-ingestion.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,13 +147,13 @@ table = pa.Table.from_pandas(df)
pq.write_table(table, 'parquet_data/creditcard.parquet')

# Import generic file loader component and Parquet-specific executor
from tfx.components import FileBasedExampleGen
from tfx.components.example_gen.component import FileBasedExampleGen
from tfx.components.example_gen.custom_executors import parquet_executor
from tfx.components.base.executor_spec import ExecutorClassSpec
from tfx.dsl.components.base.executor_spec import BeamExecutorSpec
from tfx.utils.dsl_utils import external_input

examples = external_input('parquet_data/')
executor_spec = ExecutorClassSpec(parquet_executor.Executor)
executor_spec = BeamExecutorSpec(parquet_executor.Executor)
example_gen = FileBasedExampleGen(input_base=examples, custom_executor_spec=executor_spec)

context.run(example_gen)
Expand Down Expand Up @@ -356,4 +356,4 @@ input = example_gen_pb2.Input(splits=[

examples = external_input('data')
example_gen = CsvExampleGen(input=examples, input_config=input)
```
```

0 comments on commit 2b9fedb

Please sign in to comment.