Merge pull request #97 from lsst/tickets/DM-45485

DM-45485: Add flag for ignoring constraints in schema files

python/felis/cli.py

@@ -71,7 +71,7 @@ def cli(log_level: str, log_file: str | None) -> None:
 
 
 @cli.command("create", help="Create database objects from the Felis file")
-@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
 @click.option("--schema-name", help="Alternate schema name to override Felis file")
 @click.option(
  "--initialize",
@@ -86,6 +86,7 @@ def cli(log_level: str, log_file: str | None) -> None:
 @click.option(
  "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
 )
+@click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
 @click.argument("file", type=click.File())
 def create(
  engine_url: str,
@@ -95,6 +96,7 @@ def create(
  echo: bool,
  dry_run: bool,
  output_file: IO[str] | None,
+ ignore_constraints: bool,
  file: IO,
 ) -> None:
  """Create database objects from the Felis file.
@@ -115,6 +117,8 @@ def create(
  Dry run only to print out commands instead of executing.
  output_file
  Write SQL commands to a file instead of executing.
+ ignore_constraints
+ Ignore constraints when creating tables.
  file
  Felis file to read.
  """
@@ -132,7 +136,7 @@ def create(
  dry_run = True
  logger.info("Forcing dry run for non-sqlite engine URL with no host")
 
- metadata = MetaDataBuilder(schema).build()
+ metadata = MetaDataBuilder(schema, ignore_constraints=ignore_constraints).build()
  logger.debug(f"Created metadata with schema name: {metadata.schema}")
 
  engine: Engine | MockConnection
@@ -208,7 +212,7 @@ def init_tap(
  tables are created in the database schema specified by the engine URL,
  which must be a PostgreSQL schema or MySQL database that already exists.
  """
- engine = create_engine(engine_url, echo=True)
+ engine = create_engine(engine_url)
  init_tables(
  tap_schema_name,
  tap_schemas_table,
@@ -221,7 +225,7 @@ def init_tap(
 
 
 @cli.command("load-tap", help="Load metadata from a Felis file into a TAP_SCHEMA database")
-@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
 @click.option("--schema-name", help="Alternate Schema Name for Felis file")
 @click.option("--catalog-name", help="Catalog Name for Schema")
 @click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")

python/felis/metadata.py

@@ -127,10 +127,16 @@ class MetaDataBuilder:
  Whether to apply the schema name to the metadata object.
  apply_schema_to_tables
  Whether to apply the schema name to the tables.
+ ignore_constraints
+ Whether to ignore constraints when building the metadata.
  """
 
  def __init__(
- self, schema: Schema, apply_schema_to_metadata: bool = True, apply_schema_to_tables: bool = True
+ self,
+ schema: Schema,
+ apply_schema_to_metadata: bool = True,
+ apply_schema_to_tables: bool = True,
+ ignore_constraints: bool = False,
  ) -> None:
  """Initialize the metadata builder."""
  self.schema = schema
@@ -141,6 +147,7 @@ def __init__(
  self.metadata = MetaData(schema=schema.name if apply_schema_to_metadata else None)
  self._objects: dict[str, Any] = {}
  self.apply_schema_to_tables = apply_schema_to_tables
+ self.ignore_constraints = ignore_constraints
 
  def build(self) -> MetaData:
  """Build the SQLAlchemy tables and constraints from the schema.
@@ -157,7 +164,10 @@ def build(self) -> MetaData:
  The SQLAlchemy metadata object.
  """
  self.build_tables()
- self.build_constraints()
+ if not self.ignore_constraints:
+ self.build_constraints()
+ else:
+ logger.warning("Ignoring constraints")
  return self.metadata
 
  def build_tables(self) -> None:

tests/test_cli.py

@@ -69,6 +69,25 @@ def test_create_all_dry_run(self) -> None:
  )
  self.assertEqual(result.exit_code, 0)
 
+ def test_ignore_constraints(self) -> None:
+ """Test ``--ignore-constraints`` flag of ``create`` command."""
+ url = f"sqlite:///{self.tmpdir}/tap.sqlite3"
+
+ runner = CliRunner()
+ result = runner.invoke(
+ cli,
+ [
+ "create",
+ "--schema-name=main",
+ "--ignore-constraints",
+ f"--engine-url={url}",
+ "--dry-run",
+ TEST_YAML,
+ ],
+ catch_exceptions=False,
+ )
+ self.assertEqual(result.exit_code, 0)
+
  def test_init_tap(self) -> None:
  """Test for ``init-tap`` command."""
  url = f"sqlite:///{self.tmpdir}/tap.sqlite3"

tests/test_metadata.py

@@ -25,6 +25,7 @@
 import yaml
 from sqlalchemy import (
  CheckConstraint,
+ Connection,
  Constraint,
  ForeignKeyConstraint,
  Index,
@@ -52,11 +53,11 @@ def setUp(self) -> None:
  with open(TEST_YAML) as data:
  self.yaml_data = yaml.safe_load(data)
 
- def connection(self):
+ def connection(self) -> Connection:
  """Return a connection to the database."""
  return self.engine.connect()
 
- def test_create_all(self):
+ def test_create_all(self) -> None:
  """Create all tables in the schema using the metadata object and a
  SQLite connection.
 
@@ -113,16 +114,25 @@ def _sorted_constraints(constraints: set[Constraint]) -> list[Constraint]:
  self.assertEqual(md_constraint.name, md_db_constraint.name)
  self.assertEqual(md_constraint.deferrable, md_db_constraint.deferrable)
  self.assertEqual(md_constraint.initially, md_db_constraint.initially)
- if isinstance(md_constraint, ForeignKeyConstraint):
+ self.assertEqual(
+ type(md_constraint), type(md_db_constraint), "Constraint types do not match"
+ )
+ if isinstance(md_constraint, ForeignKeyConstraint) and isinstance(
+ md_db_constraint, ForeignKeyConstraint
+ ):
  md_fk: ForeignKeyConstraint = md_constraint
  md_db_fk: ForeignKeyConstraint = md_db_constraint
  self.assertEqual(md_fk.referred_table.name, md_db_fk.referred_table.name)
  self.assertEqual(md_fk.column_keys, md_db_fk.column_keys)
- elif isinstance(md_constraint, UniqueConstraint):
+ elif isinstance(md_constraint, UniqueConstraint) and isinstance(
+ md_db_constraint, UniqueConstraint
+ ):
  md_uniq: UniqueConstraint = md_constraint
  md_db_uniq: UniqueConstraint = md_db_constraint
  self.assertEqual(md_uniq.columns.keys(), md_db_uniq.columns.keys())
- elif isinstance(md_constraint, CheckConstraint):
+ elif isinstance(md_constraint, CheckConstraint) and isinstance(
+ md_db_constraint, CheckConstraint
+ ):
  md_check: CheckConstraint = md_constraint
  md_db_check: CheckConstraint = md_db_constraint
  self.assertEqual(str(md_check.sqltext), str(md_db_check.sqltext))
@@ -139,7 +149,7 @@ def _sorted_constraints(constraints: set[Constraint]) -> list[Constraint]:
  self.assertEqual(md_index.name, md_db_index.name)
  self.assertEqual(md_index.columns.keys(), md_db_index.columns.keys())
 
- def test_builder(self):
+ def test_builder(self) -> None:
  """Test that the information in the metadata object created by the
  builder matches the data in the Felis schema used to create it.
  """
@@ -188,7 +198,7 @@ def test_builder(self):
  for primary_key in primary_keys:
  self.assertTrue(md_table.columns[primary_key].primary_key)
 
- def test_timestamp(self):
+ def test_timestamp(self) -> None:
  """Test that the `timestamp` datatype is created correctly."""
  for precision in [None, 6]:
  col = dm.Column(
@@ -210,6 +220,24 @@ def test_timestamp(self):
  self.assertEqual(mysql_timestamp.timezone, False)
  self.assertEqual(mysql_timestamp.fsp, precision)
 
+ def test_ignore_constraints(self) -> None:
+ """Test that constraints are not created when the
+ ``ignore_constraints`` flag is set on the metadata builder.
+ """
+ schema = Schema.model_validate(self.yaml_data)
+ schema.name = "main"
+ builder = MetaDataBuilder(schema, ignore_constraints=True)
+ md = builder.build()
+ for table in md.tables.values():
+ non_primary_key_constraints = [
+ c for c in table.constraints if not isinstance(c, PrimaryKeyConstraint)
+ ]
+ self.assertEqual(
+ len(non_primary_key_constraints),
+ 0,
+ msg=f"Table {table.name} has non-primary key constraints defined",
+ )
+
 
 if __name__ == "__main__":
  unittest.main()

types.txt

@@ -1,4 +1,3 @@
 types-PyYAML
 types-click
-types-pkg_resources
 types-Deprecated