update SQL Directory env and collection Extent

Author: vincentsarago

.pre-commit-config.yaml

@@ -26,7 +26,9 @@ repos:
           - toml
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.812
+    rev: v0.991
     hooks:
       - id: mypy
         language_version: python
+        # No reason to run if only tests have changed. They intentionally break typing.
+        exclude: tests/.*

tests/conftest.py

@@ -70,12 +70,12 @@ def app(database_url, monkeypatch):
     monkeypatch.setenv("TIPG_TABLE_CONFIG__public_my_data_alt__pk", "id")
     monkeypatch.setenv("TIPG_TABLE_CONFIG__public_landsat__geomcol", "geom")
 
-    # monkeypatch.setenv("TIPG_FUNCTIONS_DIRECTORY", os.path.join(DATA_DIR, "functions"))
+    # monkeypatch.setenv("TIPG_CUSTOM_SQL_DIRECTORY", os.path.join(DATA_DIR, "functions"))
 
     # OGC Tiles Settings
     monkeypatch.setenv("TIPG_DEFAULT_MINZOOM", str(5))
     monkeypatch.setenv("TIPG_DEFAULT_MAXZOOM", str(12))
-    monkeypatch.setenv("TIPG_FUNCTIONS_DIRECTORY", "tests/fixtures/functions")
+    monkeypatch.setenv("TIPG_CUSTOM_SQL_DIRECTORY", "tests/fixtures/functions")
 
     from tipg.main import app, postgres_settings
 

tests/routes/test_collections.py

@@ -51,7 +51,7 @@ def test_collections_search(app):
 
     response = app.get("/collections", params={"bbox": "-180,81,180,87"})
     body = response.json()
-    assert body["numberMatched"] == 9
+    assert body["numberMatched"] == 10
     ids = [x["id"] for x in body["collections"]]
     assert "public.nongeo_data" not in ids
     assert "public.canada" not in ids
@@ -74,18 +74,17 @@ def test_collections_search(app):
 
     response = app.get("/collections", params={"datetime": "2004-12-31T23:59:59Z/.."})
     body = response.json()
-    assert body["numberMatched"] == 1
+    assert body["numberMatched"] == 2
     ids = [x["id"] for x in body["collections"]]
-
-    assert ["public.my_data_alt"] == ids
+    assert ["public.my_data", "public.my_data_alt"] == ids
 
     response = app.get(
         "/collections", params={"datetime": "2004-01-01T00:00:00Z/2004-12-31T23:59:59Z"}
     )
     body = response.json()
-    assert body["numberMatched"] == 2
+    assert body["numberMatched"] == 1
     ids = [x["id"] for x in body["collections"]]
-    assert ["public.my_data", "public.nongeo_data"] == ids
+    assert ["public.nongeo_data"] == ids
 
 
 def test_collections_excludes(app_excludes):

tipg/db.py

@@ -1,31 +1,36 @@
 """tipg.db: database events."""
 
-from pathlib import Path
+import os
+import pathlib
 from typing import Any, Optional
 
 import orjson
 from buildpg import asyncpg
 
 from tipg.dbmodel import get_collection_index
+from tipg.errors import FunctionDirectoryDoesNotExist
 from tipg.settings import PostgresSettings
 
 from fastapi import FastAPI
 
+custom_sql: list[pathlib.Path] = []
+user_sql_dir = os.environ.get("TIPG_CUSTOM_SQL_DIRECTORY", None)
+if user_sql_dir:
+    if not pathlib.Path(user_sql_dir).exists():
+        raise FunctionDirectoryDoesNotExist
+
+    custom_sql = list(pathlib.Path(user_sql_dir).glob("*.sql"))
 
-async def con_init(
-    conn,
-    settings: Optional[PostgresSettings] = None,
-):
-    """Use json for json returns."""
-    if not settings:
-        settings = PostgresSettings()
 
+async def con_init(conn):
+    """Use orjson encoder/decoder and register custom SQL functions."""
     await conn.set_type_codec(
         "json", encoder=orjson.dumps, decoder=orjson.loads, schema="pg_catalog"
     )
     await conn.set_type_codec(
         "jsonb", encoder=orjson.dumps, decoder=orjson.loads, schema="pg_catalog"
     )
+
     await conn.execute(
         """
             SELECT set_config(
@@ -35,13 +40,14 @@ async def con_init(
                 );
             """
     )
-    if (
-        settings.tipg_functions_directory
-        and Path(settings.tipg_functions_directory).exists()
-    ):
-        for sqlfile in Path(settings.tipg_functions_directory).glob("**/*.sql"):
+
+    # Register custom SQL functions/table/views
+    if custom_sql:
+        for sqlfile in custom_sql:
             await conn.execute(sqlfile.read_text())
-    dbcatalogsql = Path("tipg/sql/dbcatalog.sql")
+
+    # Register TiPG functions
+    dbcatalogsql = pathlib.Path(__package__) / "sql" / "dbcatalog.sql"
     await conn.execute(dbcatalogsql.read_text())
 
 

tipg/dbmodel.py

@@ -22,7 +22,7 @@
 )
 from tipg.filter.evaluate import to_filter
 from tipg.filter.filters import bbox_to_wkt
-from tipg.model import Extent, Spatial, Temporal
+from tipg.model import Extent
 from tipg.settings import TableSettings, TileSettings
 
 tile_settings = TileSettings()
@@ -135,9 +135,7 @@ def json_type(self) -> str:
     @property
     def is_geometry(self) -> bool:
         """Returns true if this property is a geometry column."""
-        if self.geometry_type:
-            return True
-        return False
+        return self.type in ("geometry", "geography")
 
     @property
     def is_datetime(self) -> bool:
@@ -170,26 +168,63 @@ class Collection(BaseModel):
     default_tms: str = tile_settings.default_tms
 
     @property
-    def geometry_columns(self):
-        """Return geometry columns."""
-        return [c for c in self.properties if c.is_geometry]
+    def extent(self) -> Optional[Extent]:
+        """Return extent."""
+        extent = {}
+        if cols := self.geometry_columns:
+            if len(cols) == 1:
+                bbox = [cols[0].bounds]
+            else:
+                minx, miny, maxx, maxy = zip(*[col.bounds for col in cols])
+                bbox = [
+                    [min(minx), min(miny), max(maxx), max(maxy)],
+                    *[col.bounds for col in cols],
+                ]
+
+            extent["spatial"] = {
+                "bbox": bbox,
+                "crs": f"http://www.opengis.net/def/crs/EPSG/0/{cols[0].srid}",
+            }
+
+        if cols := self.datetime_columns:
+            cols = [col for col in cols if col.mindt or col.maxdt]
+
+            intervals = []
+            if len(cols) == 1:
+                if cols[0].mindt or cols[0].maxdt:
+                    intervals = [[cols[0].mindt, cols[0].maxdt]]
+
+            else:
+                mindt = [col.mindt for col in cols if col.mindt]
+                maxdt = [col.maxdt for col in cols if col.maxdt]
+                intervals = [
+                    [min(mindt), max(maxdt)],
+                    *[[col.mindt, col.maxdt] for col in cols if col.mindt or col.maxdt],
+                ]
+
+            if intervals:
+                extent["temporal"] = {"interval": intervals}
+
+        if extent:
+            return Extent(**extent)
+
+        return None
 
     @property
-    def extent(self):
-        """Return extent."""
-        spatial = None
-        temporal = None
-        if self.geometry_column and self.geometry_column.bounds:
-            spatial = Spatial(bbox=[self.geometry_column.bounds], crs=self.crs)
-        if (
-            self.datetime_column
-            and self.datetime_column.mindt
-            and self.datetime_column.maxdt
-        ):
-            temporal = Temporal(
-                interval=[[self.datetime_column.mindt, self.datetime_column.maxdt]]
-            )
-        return Extent(spatial=spatial, temporal=temporal)
+    def bounds(self) -> Optional[List[float]]:
+        """Return spatial bounds from collection extent."""
+        if self.extent and self.extent.spatial:
+            return self.extent.spatial.bbox[0]
+
+        return None
+
+    @property
+    def dt_bounds(self) -> Optional[List[str]]:
+        """Return temporal bounds from collection extent."""
+        if self.extent and self.extent.temporal:
+            return self.extent.temporal.interval[0]
+
+        return None
 
     @property
     def crs(self):
@@ -198,46 +233,43 @@ def crs(self):
             return f"http://www.opengis.net/def/crs/EPSG/0/{self.geometry_column.srid}"
 
     @property
-    def datetime_columns(self):
+    def geometry_columns(self) -> List[Column]:
+        """Return geometry columns."""
+        return [c for c in self.properties if c.is_geometry]
+
+    @property
+    def datetime_columns(self) -> List[Column]:
         """Return datetime columns."""
         return [c for c in self.properties if c.is_datetime]
 
-    def get_datetime_column(self, name: Optional[str] = None) -> Optional[Column]:
-        """Return the Column for either the passed in tstz column or the first tstz column."""
-        if not self.datetime_columns:
+    def get_geometry_column(self, name: Optional[str] = None) -> Optional[Column]:
+        """Return the name of the first geometry column."""
+        if (not self.geometry_columns) or (name and name.lower() == "none"):
             return None
 
         if name is None:
-            return self.datetime_column
+            return self.geometry_column
 
-        for col in self.datetime_columns:
+        for col in self.geometry_columns:
             if col.name == name:
                 return col
 
         return None
 
-    def get_geometry_column(self, name: Optional[str] = None) -> Optional[Column]:
-        """Return the name of the first geometry column."""
-        if (not self.geometry_columns) or (name and name.lower() == "none"):
+    def get_datetime_column(self, name: Optional[str] = None) -> Optional[Column]:
+        """Return the Column for either the passed in tstz column or the first tstz column."""
+        if not self.datetime_columns:
             return None
 
         if name is None:
-            return self.geometry_column
+            return self.datetime_column
 
-        for col in self.geometry_columns:
+        for col in self.datetime_columns:
             if col.name == name:
                 return col
 
         return None
 
-    @property
-    def bounds(self) -> Optional[List[float]]:
-        """Return bounds from collection extent."""
-        if self.extent and self.extent.spatial:
-            return self.extent.spatial.bbox[0]
-
-        return None
-
     @property
     def id_column_info(self) -> Column:  # type: ignore
         """Return Column for a unique identifier."""
@@ -913,6 +945,7 @@ async def get_collection_index(  # noqa: C901
                         or datetime_column is None
                     ):
                         datetime_column = c
+
                 if c.get("type") in ("geometry", "geography"):
                     if (
                         table_conf.get("geomcol") == c["name"]

tipg/factory.py

@@ -87,20 +87,17 @@ def write(self, line: str):
         yield writer.writerow(row)
 
 
-def s_intersects(bbox: List[float], spatial_extent: List[List[float]]) -> bool:
+def s_intersects(bbox: List[float], spatial_extent: List[float]) -> bool:
     """Check if bbox intersects with spatial extent."""
-    for ext in spatial_extent:
-        if (
-            (bbox[0] < ext[2])
-            and (bbox[2] > ext[0])
-            and (bbox[3] > ext[1])
-            and (bbox[1] < ext[3])
-        ):
-            return True
-    return False
+    return (
+        (bbox[0] < spatial_extent[2])
+        and (bbox[2] > spatial_extent[0])
+        and (bbox[3] > spatial_extent[1])
+        and (bbox[1] < spatial_extent[3])
+    )
 
 
-def t_intersects(interval: List[str], temporal_extent: List[List[str]]) -> bool:
+def t_intersects(interval: List[str], temporal_extent: List[str]) -> bool:
     """Check if dates intersect with temporal extent."""
     if len(interval) == 1:
         start = end = parse_rfc3339(interval[0])
@@ -109,22 +106,22 @@ def t_intersects(interval: List[str], temporal_extent: List[List[str]]) -> bool:
         start = parse_rfc3339(interval[0]) if not interval[0] in ["..", ""] else None
         end = parse_rfc3339(interval[1]) if not interval[1] in ["..", ""] else None
 
-    for (mint, maxt) in temporal_extent:
-        min_ext = parse_rfc3339(mint) if mint is not None else None
-        max_ext = parse_rfc3339(maxt) if maxt is not None else None
+    mint, maxt = temporal_extent
+    min_ext = parse_rfc3339(mint) if mint is not None else None
+    max_ext = parse_rfc3339(maxt) if maxt is not None else None
 
-        if len(interval) == 1:
-            if start == min_ext or start == max_ext:
-                return True
+    if len(interval) == 1:
+        if start == min_ext or start == max_ext:
+            return True
 
-        if not start:
-            return max_ext <= end or min_ext <= end
+    if not start:
+        return max_ext <= end or min_ext <= end
 
-        elif not end:
-            return min_ext >= start or max_ext >= start
+    elif not end:
+        return min_ext >= start or max_ext >= start
 
-        else:
-            return min_ext >= start and max_ext <= end
+    else:
+        return min_ext >= start and max_ext <= end
 
     return False
 
@@ -447,21 +444,17 @@ def collections(
                 collections_list = [
                     collection
                     for collection in collections_list
-                    if collection.extent is not None
-                    and collection.extent.spatial is not None
-                    and s_intersects(bbox_filter, collection.extent.spatial.bbox)
+                    if collection.bounds is not None
+                    and s_intersects(bbox_filter, collection.bounds)
                 ]
 
             # datetime filter
             if datetime_filter is not None:
                 collections_list = [
                     collection
                     for collection in collections_list
-                    if collection.extent is not None
-                    and collection.extent.temporal is not None
-                    and t_intersects(
-                        datetime_filter, collection.extent.temporal.interval
-                    )
+                    if collection.dt_bounds is not None
+                    and t_intersects(datetime_filter, collection.dt_bounds)
                 ]
 
             matched_items = len(collections_list)