Merge pull request #90 from lsst/tickets/DM-45263

DM-45263: Add new tap_schema module

Makefile

@@ -20,6 +20,9 @@ print_target:
 build:
  @uv pip install --force-reinstall --no-deps -e .
 
+deps:
+ @uv pip install --upgrade -r requirements.txt
+
 docs:
  @rm -rf docs/dev/internals docs/_build
  @tox -e docs

docs/changes/DM-45263.feature.rst

@@ -0,0 +1,3 @@
+Added a new ``tap_schema_`` module designed to deprecate and eventually replace the ``tap`` module.
+This module provides utilities for translating a Felis schema into a TAP_SCHEMA representation.
+The command ``felis load-tap-schema`` can be used to activate this functionality.

docs/dev/internals.rst

@@ -7,33 +7,41 @@ Python API
 .. automodapi:: felis.datamodel
  :include-all-objects:
 
-.. automodapi:: felis.metadata
+.. automodapi:: felis.db.dialects
  :include-all-objects:
  :no-inheritance-diagram:
 
-.. automodapi:: felis.tap
+.. automodapi:: felis.db.sqltypes
  :include-all-objects:
  :no-inheritance-diagram:
 
-.. automodapi:: felis.types
- :include-all-objects:
-
-.. automodapi:: felis.db.dialects
+.. automodapi:: felis.db.utils
  :include-all-objects:
  :no-inheritance-diagram:
 
-.. automodapi:: felis.db.sqltypes
+.. automodapi:: felis.db.variants
  :include-all-objects:
  :no-inheritance-diagram:
 
-.. automodapi:: felis.db.utils
+.. automodapi:: felis.metadata
  :include-all-objects:
  :no-inheritance-diagram:
 
-.. automodapi:: felis.db.variants
+.. automodapi:: felis.tap
  :include-all-objects:
  :no-inheritance-diagram:
 
+.. automodapi:: felis.tap_schema
+ :include-all-objects:
+ :no-inheritance-diagram:
+
 .. automodapi:: felis.tests.postgresql
  :include-all-objects:
  :no-inheritance-diagram:
+
+.. automodapi:: felis.tests.utils
+ :include-all-objects:
+ :no-inheritance-diagram:
+
+.. automodapi:: felis.types
+ :include-all-objects:

docs/documenteer.toml

@@ -21,6 +21,8 @@ nitpick_ignore = [
  ["py:class", "sqlalchemy.orm.decl_api.Base"],
  ["py:class", "sqlalchemy.engine.mock.MockConnection"],
  ["py:class", "pydantic.main.BaseModel"],
+ ["py:exc", "pydantic.ValidationError"],
+ ["py:exc", "yaml.YAMLError"]
 ]
 nitpick_ignore_regex = [
  # Bug in autodoc_pydantic.
@@ -29,5 +31,6 @@ nitpick_ignore_regex = [
 python_api_dir = "dev/internals"
 
 [sphinx.intersphinx.projects]
-python = "https://docs.python.org/3/"
-sqlalchemy = "https://docs.sqlalchemy.org/en/latest/"
+python = "https://docs.python.org/3"
+sqlalchemy = "https://docs.sqlalchemy.org/en/latest"
+lsst = "https://pipelines.lsst.io/v/weekly"

docs/user-guide/datatypes.rst

@@ -74,7 +74,7 @@ The following table shows these mapping:
 +-----------+---------------+----------+------------------+--------------+
 | unicode | NVARCHAR | NVARCHAR | VARCHAR | unicodeChar |
 +-----------+---------------+----------+------------------+--------------+
-| text | TEXT | LONGTEXT | TEXT | uncodeChar |
+| text | TEXT | LONGTEXT | TEXT | char  |
 +-----------+---------------+----------+------------------+--------------+
 | binary | BLOB | LONGBLOB | BYTEA | unsignedByte |
 +-----------+---------------+----------+------------------+--------------+

pyproject.toml

@@ -26,7 +26,8 @@ dependencies = [
  "click >= 7",
  "pyyaml >= 6",
  "pydantic >= 2, < 3",
- "lsst-utils"
+ "lsst-utils",
+ "lsst-resources"
 ]
 requires-python = ">=3.11.0"
 dynamic = ["version"]
@@ -55,7 +56,7 @@ zip-safe = true
 license-files = ["COPYRIGHT", "LICENSE"]
 
 [tool.setuptools.package-data]
-"felis" = ["py.typed"]
+"felis" = ["py.typed", "schemas/*.yaml"]
 
 [tool.setuptools.dynamic]
 version = { attr = "lsst_versions.get_lsst_version" }

python/felis/cli.py

@@ -23,22 +23,21 @@
 
 from __future__ import annotations
 
-import io
 import logging
 from collections.abc import Iterable
 from typing import IO
 
 import click
-import yaml
 from pydantic import ValidationError
 from sqlalchemy.engine import Engine, create_engine, make_url
-from sqlalchemy.engine.mock import MockConnection
+from sqlalchemy.engine.mock import MockConnection, create_mock_engine
 
 from . import __version__
 from .datamodel import Schema
-from .db.utils import DatabaseContext
+from .db.utils import DatabaseContext, is_mock_url
 from .metadata import MetaDataBuilder
 from .tap import Tap11Base, TapLoadingVisitor, init_tables
+from .tap_schema import DataLoader, TableManager
 
 __all__ = ["cli"]
 
@@ -107,7 +106,7 @@ def create(
  dry_run: bool,
  output_file: IO[str] | None,
  ignore_constraints: bool,
- file: IO,
+ file: IO[str],
 ) -> None:
  """Create database objects from the Felis file.
 
@@ -133,8 +132,7 @@ def create(
  Felis file to read.
  """
  try:
- yaml_data = yaml.safe_load(file)
- schema = Schema.model_validate(yaml_data, context={"id_generation": ctx.obj["id_generation"]})
+ schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
  url = make_url(engine_url)
  if schema_name:
  logger.info(f"Overriding schema name with: {schema_name}")
@@ -261,7 +259,7 @@ def load_tap(
  tap_keys_table: str,
  tap_key_columns_table: str,
  tap_schema_index: int,
- file: io.TextIOBase,
+ file: IO[str],
 ) -> None:
  """Load TAP metadata from a Felis file.
 
@@ -304,8 +302,7 @@ def load_tap(
  The data will be loaded into the TAP_SCHEMA from the engine URL. The
  tables must have already been initialized or an error will occur.
  """
- yaml_data = yaml.load(file, Loader=yaml.SafeLoader)
- schema = Schema.model_validate(yaml_data)
+ schema = Schema.from_stream(file)
 
  tap_tables = init_tables(
  tap_schema_name,
@@ -345,6 +342,79 @@ def load_tap(
  tap_visitor.visit_schema(schema)
 
 
+@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
+@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
+@click.option(
+ "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
+)
+@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema in this environment")
+@click.option("--dry-run", is_flag=True, help="Execute dry run only. Does not insert any data.")
+@click.option("--echo", is_flag=True, help="Print out the generated insert statements to stdout")
+@click.option("--output-file", type=click.Path(), help="Write SQL commands to a file")
+@click.argument("file", type=click.File())
+@click.pass_context
+def load_tap_schema(
+ ctx: click.Context,
+ engine_url: str,
+ tap_schema_name: str,
+ tap_tables_postfix: str,
+ tap_schema_index: int,
+ dry_run: bool,
+ echo: bool,
+ output_file: str | None,
+ file: IO[str],
+) -> None:
+ """Load TAP metadata from a Felis file.
+
+ Parameters
+ ----------
+ engine_url
+ SQLAlchemy Engine URL.
+ tap_tables_postfix
+ Postfix which is applied to standard TAP_SCHEMA table names.
+ tap_schema_index
+ TAP_SCHEMA index of the schema in this environment.
+ dry_run
+ Execute dry run only. Does not insert any data.
+ echo
+ Print out the generated insert statements to stdout.
+ output_file
+ Output file for writing generated SQL.
+ file
+ Felis file to read.
+
+ Notes
+ -----
+ The TAP_SCHEMA database must already exist or the command will fail. This
+ command will not initialize the TAP_SCHEMA tables.
+ """
+ url = make_url(engine_url)
+ engine: Engine | MockConnection
+ if dry_run or is_mock_url(url):
+ engine = create_mock_engine(url, executor=None)
+ else:
+ engine = create_engine(engine_url)
+ mgr = TableManager(
+ engine=engine,
+ apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
+ schema_name=tap_schema_name,
+ table_name_postfix=tap_tables_postfix,
+ )
+
+ schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
+
+ DataLoader(
+ schema,
+ mgr,
+ engine,
+ tap_schema_index=tap_schema_index,
+ dry_run=dry_run,
+ print_sql=echo,
+ output_path=output_file,
+ ).load()
+
+
 @cli.command("validate", help="Validate one or more Felis YAML files")
 @click.option(
  "--check-description", is_flag=True, help="Check that all objects have a description", default=False
@@ -372,7 +442,7 @@ def validate(
  check_redundant_datatypes: bool,
  check_tap_table_indexes: bool,
  check_tap_principal: bool,
- files: Iterable[io.TextIOBase],
+ files: Iterable[IO[str]],
 ) -> None:
  """Validate one or more felis YAML files.
 
@@ -406,9 +476,8 @@ def validate(
  file_name = getattr(file, "name", None)
  logger.info(f"Validating {file_name}")
  try:
- data = yaml.load(file, Loader=yaml.SafeLoader)
- Schema.model_validate(
- data,
+ Schema.from_stream(
+ file,
  context={
  "check_description": check_description,
  "check_redundant_datatypes": check_redundant_datatypes,