refactor (#15)

* create stac-sprint dev branch

* improve local development (#12)

* use postgis official image, update migration command

* remove comment

* update dockerfile

* improve makefile

* update readme

* add pre-commit, make pre-commit happy

* update pipfile

* add settings injection (#13)

* add settings injection

* update cicd

* decouple transactions and core endpoints (#14)

* define base clients, add transaction client (#16)

* define base clients, add transaction client

* use absolute imports

* cleanup

* add transactions router factory (#17)

* add transactions router factory

* add create_app method

* add collection router factory (#19)

* improve postgres clients

* rename api models

* add collection router factory

* add item collection route

* add item router factory (#20)

* standardize input (#21)

* whoops

* remove apiresponse

* add empty request

* update endpoint

* fix some things

* configure api extensions (#22)

* create request body models

* add stac api extensions to api settings

* helper method to check if api extension is enabled

* parametrize transaction extension

* parametrize query extension

* parametrize context extension

* parametrize field extension

* set model base

* simplify app creation (#23)

* add tiles add on, consolidate core endpoints (#24)

* template simple tiles router

* update dockerfile

* update pipfile

* add more tiles links

* finish tiles client

* create stac router

* request kwargs (#25)

* pass through request

* rearrange

* remove dependency

* remove references to dependency

* fix everything

* unify core spec (#26)

* unify core spec

* update tests

* cleanup

* update docstrings

* add get search to router factory (#27)

* update titiler version, update openapi

* fix item uri

* updates (#28)

* update dependencies

* updates

* update dockerfile

* add tiles link

* add wmts link, add link titles

* add api module (#29)

* update tests, fix bugs (#30)

* don't mutate orm objects

* oops

* update tests

* remove echo

* add api test cases

* add api test cases

* fix test

* update conformance (#31)

* add conformance routes to factory

* remove comment

* update test

* move liveliness probe (#32)

* enable sql echo

* optimize sql (#33)

* faster count

* remove a select

* optimize fetching collection children

* remove id from default sort

* use a join

* fix paging

* raise notfounderror

* update test

* dont raise

* update titiler (#34)

* update branch rules

Author: geospatial-jeff

.pre-commit-config.yaml

@@ -1,6 +1,44 @@
 repos:
-  - repo: https://github.com/python/black
-    rev: stable
-    hooks:
-      - id: black
-        language_version: python3.8
+    -
+        repo: https://github.com/pre-commit/mirrors-isort
+        rev: v4.3.21
+        hooks:
+            - id: isort
+              language_version: python3.7
+    -
+        repo: https://github.com/psf/black
+        rev: stable
+        hooks:
+            - id: black
+              args: ['--safe']
+              language_version: python3.7
+    -
+        repo: https://github.com/pre-commit/pre-commit-hooks
+        rev: v2.4.0
+        hooks:
+            - id: flake8
+              language_version: python3.7
+              args: [
+                  # E501 let black handle all line length decisions
+                  # W503 black conflicts with "line break before operator" rule
+                  # E203 black conflicts with "whitespace before ':'" rule
+                  '--ignore=E501,W503,E203']
+    -
+        repo: https://github.com/chewse/pre-commit-mirrors-pydocstyle
+        # 2.1.1
+        rev: 22d3ccf6cf91ffce3b16caa946c155778f0cb20f
+        hooks:
+            - id: pydocstyle
+              language_version: python3.7
+              args: [
+                 # Check for docstring presence only
+                 '--select=D1',
+                 # Don't require docstrings for tests
+                 '--match=(?!test).*\.py']
+    -
+        repo: https://github.com/pre-commit/mirrors-mypy
+        rev: v0.770
+        hooks:
+            -   id: mypy
+                language_version: python3.7
+                args: [--no-strict-optional, --ignore-missing-imports]

Dockerfile

@@ -2,15 +2,12 @@ FROM python:3.8-slim
 
 # Any python libraries that require system libraries to be installed will likely
 # need the following packages in order to build
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    libffi-dev \
-    libssl-dev \
-    git
+RUN apt-get update && apt-get install -y build-essential git
 
 RUN pip install pipenv
 ENV PIPENV_NOSPIN=true
 ENV PIPENV_HIDE_EMOJIS=true
+ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 
 ARG install_dev_dependencies=true
 
@@ -24,8 +21,5 @@ COPY . ./
 ENV APP_HOST=0.0.0.0
 ENV APP_PORT=80
 
-ENV RELOAD=''
-
-
 ENTRYPOINT ["pipenv", "run"]
-CMD uvicorn stac_api.app:app --host=${APP_HOST} --port=${APP_PORT} ${RELOAD:+--reload}
+CMD uvicorn stac_api.app:app --host=${APP_HOST} --port=${APP_PORT} --reload

Makefile

@@ -14,7 +14,15 @@ run_docker = docker run -it --rm \
 				--env POSTGRES_HOST_WRITER=host.docker.internal \
 				--env POSTGRES_PORT=5432 \
 				--env ENVIRONMENT=development \
-				arturo-stac-api_app
+				arturo-ai/stac-api:latest
+
+.PHONY: image
+image:
+	docker build -t arturo-ai/stac-api:latest .
+
+.PHONY: docker-run
+docker-run: image
+	$(run_docker)
 
 .PHONY: docker-shell
 docker-shell:

Pipfile

@@ -13,12 +13,16 @@ sqlalchemy = "*"
 geoalchemy2 = "<0.8.0"
 sqlakeyset = "*"
 stac-pydantic = ">=1.1.3"
+pydantic = {extras=["dotenv"], version="*"}
+cogeo-mosaic = "==3.0a10"
+titiler = "==0.1a0"
 
 [dev-packages]
 pytest = "*"
 pytest-cov = "*"
 pytest-asyncio = "*"
 requests = "*"
+pre-commit = "*"
 
 [requires]
 python_version = "3.8"

Pipfile.lock

@@ -18,13 +18,26 @@ FastAPI/postgres implementation of the [STAC API specification](https://github.c
 ```
 
 ## Local Development
-Use docker-compose to deploy the application, migrate the database, and ingest an example collection:
-```
+Use docker-compose to deploy the application, migrate the database, and ingest some example data:
+```bash
 docker-compose build
 docker-compose up
 ```
 
-Run tests (the `docker-compose` stack must be running):
+For local development it is often more convenient to run the application outside of docker-compose:
+```bash
+make docker-run
 ```
+
+
+### Testing
+The database container provided by the docker-compose stack must be running.  Run all tests:
+```bash
 make test
+```
+
+Run individual tests by running pytest within the docker container:
+```bash
+make docker-shell
+$ pytest -v
 ```

README.md

@@ -1,12 +1,11 @@
+"""Migration environment."""
 import os
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
+from sqlalchemy import engine_from_config, pool
 
 from alembic import context
 
-
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
@@ -26,15 +25,16 @@
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
+
 def get_connection_url() -> str:
     """
     Get connection URL from environment variables (see `.env.example`)
     """
-    postgres_user = os.environ['POSTGRES_USER']
-    postgres_pass = os.environ['POSTGRES_PASS']
-    postgres_host = os.environ['POSTGRES_HOST']
-    postgres_port = os.environ['POSTGRES_PORT']
-    postgres_dbname = os.environ['POSTGRES_DBNAME']
+    postgres_user = os.environ["POSTGRES_USER"]
+    postgres_pass = os.environ["POSTGRES_PASS"]
+    postgres_host = os.environ["POSTGRES_HOST"]
+    postgres_port = os.environ["POSTGRES_PORT"]
+    postgres_dbname = os.environ["POSTGRES_DBNAME"]
     return f"postgresql://{postgres_user}:{postgres_pass}@{postgres_host}:{postgres_port}/{postgres_dbname}"
 
 
@@ -70,17 +70,13 @@ def run_migrations_online():
 
     """
     configuration = config.get_section(config.config_ini_section)
-    configuration['sqlalchemy.url'] = get_connection_url()
+    configuration["sqlalchemy.url"] = get_connection_url()
     connectable = engine_from_config(
-        configuration,
-        prefix="sqlalchemy.",
-        poolclass=pool.NullPool,
+        configuration, prefix="sqlalchemy.", poolclass=pool.NullPool,
     )
 
     with connectable.connect() as connection:
-        context.configure(
-            connection=connection, target_metadata=target_metadata
-        )
+        context.configure(connection=connection, target_metadata=target_metadata)
 
         with context.begin_transaction():
             context.run_migrations()

alembic/env.py

@@ -4,13 +4,12 @@
 Revises: 
 Create Date: 2020-02-09 13:03:09.336631
 
-"""
-from alembic import op
+"""  # noqa
 import sqlalchemy as sa
-
 from sqlalchemy.dialects.postgresql import JSONB
-from geoalchemy2.types import Geometry
 
+from alembic import op
+from geoalchemy2.types import Geometry
 
 # revision identifiers, used by Alembic.
 revision = "131aab4d9e49"
@@ -20,6 +19,7 @@
 
 
 def upgrade():
+    """upgrade to this revision"""
     op.execute("CREATE SCHEMA data")
     op.execute("CREATE EXTENSION IF NOT EXISTS postgis")
 
@@ -67,6 +67,7 @@ def upgrade():
 
 
 def downgrade():
+    """downgrade to previous revision"""
     op.execute("DROP TABLE data.items")
     op.execute("DROP TABLE data.collections")
     op.execute("DROP TABLE data.paging_tokens")

alembic/versions/131aab4d9e49_create_tables.py

@@ -26,19 +26,15 @@ services:
 
   database:
     container_name: stac-db
-    image: kartoza/postgis:latest
+    image: postgis/postgis:12-3.0
     environment:
       - POSTGRES_USER=username
-      - POSTGRES_PASS=password
-      - POSTGRES_DBNAME=postgis
-      - POSTGRES_HOST=localhost
-      - POSTGRES_PORT=5432
-      - ALLOW_IP_RANGE=0.0.0.0/0
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=postgis
     ports:
       - "5432:5432"
 
   migration:
-#    image: stac-api:latest
     build:
       context: .
       dockerfile: Dockerfile
@@ -50,8 +46,6 @@ services:
       - POSTGRES_HOST=host.docker.internal
       - POSTGRES_PORT=5432
     command: >
-      bash -c "git clone https://github.com/vishnubob/wait-for-it.git &&
-               ./wait-for-it/wait-for-it.sh localhost:5432 -- alembic upgrade head &&
-               python scripts/ingest_joplin.py"
+      bash -c "sleep 10 && alembic upgrade head && python scripts/ingest_joplin.py"
     depends_on:
       - database

docker-compose.yml

@@ -1,13 +1,15 @@
+"""Ingest sample data during docker-compose"""
+
 from urllib.parse import urljoin
 
 import requests
 
-
 bucket = "arturo-stac-api-test-data"
 app_host = "http://host.docker.internal:8081"
 
 
 def ingest_joplin_data():
+    """ingest data"""
     r = requests.get(f"https://{bucket}.s3.amazonaws.com/joplin/collection.json")
     collection = r.json()