Skip to content

Commit 95265d0

Browse files
bitnerkylebarrongeospatial-jeff
authored
PGStac (#126)
* add .envrc to .*ignore * submodules all as one package * submodules all as one package * use namespace packages * remove setup.py from pgrest * update cicd workflow * cherry pick pgstac branch to updated package layout * allow async endpoints * core modifications needed for pgstac * cleanup * cleanup * setup console scripts * get transaction api working * bring in pgstac * remove old pgrest * bring Dockerfile over * cherrypicks * add removed readme back * linting, add gzip middleware * move mangum to awslambda extras * sorts * fixes for precommit * fixes for precommit * uggh pydocstyle * pgstac tests and fixes from tests * get test runners working * linting * tag for db compatible with pgstac * db docker uses different ENV vars * linted too hard * test cleanup * docker env fixes, documentation * docker env fixes, documentation * switch from gzip to brotli * Update stac_fastapi/api/stac_fastapi/api/app.py Co-authored-by: Kyle Barron <kylebarron2@gmail.com> * Update stac_fastapi/pgstac/setup.py Co-authored-by: Kyle Barron <kylebarron2@gmail.com> * Update stac_fastapi/pgstac/README.md Co-authored-by: Kyle Barron <kylebarron2@gmail.com> * fix issues in links for base_url and items relations type * fix docstring quotes * add item_id, collection_id in response to #138 * update pypgstac version * fix parent link * change id key to items_id in delete_item * update version of pgstac * Update docker-compose.yml call pgstac app as python module rather than using the cli command Co-authored-by: Jeff Albrecht <jeff@arturo.ai> * Update stac_fastapi/pgstac/README.md Co-authored-by: Jeff Albrecht <jeff@arturo.ai> * Update stac_fastapi/pgstac/stac_fastapi/pgstac/models/links.py Co-authored-by: Jeff Albrecht <jeff@arturo.ai> * Update stac_fastapi/pgstac/tests/clients/test_postgres.py Co-authored-by: Jeff Albrecht <jeff@arturo.ai> * Update stac_fastapi/pgstac/tests/clients/test_postgres.py Co-authored-by: Jeff Albrecht <jeff@arturo.ai> * Apply suggestions from code review remove unused load_test_data fixtures Co-authored-by: Jeff Albrecht <jeff@arturo.ai> * fixes from code review. remove unused/debug logging. remove unused functions. add type hints. * add import for asyncpg.pool for type hint * add default for initial pool and connection value * use request.json() because we are adding in the paging token to the POST request * linting fixes * make docker-compose use reload=false for uvicorn to avoid any issues with venvs * increase number of connections on pg instance, only mount scripts and stac_fastapi directories * move test data and add test data to both pgstac and sqlalchemy in docker, make no collections return empty array * fix test for collections returning empty array rather than not found * add exclude_unset=True when dumping data to json to add in transactions extension Co-authored-by: Kyle Barron <kylebarron2@gmail.com> Co-authored-by: Jeff Albrecht <jeff@arturo.ai>
1 parent f6a9007 commit 95265d0

49 files changed

Lines changed: 5298 additions & 193 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/cicd.yaml‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,11 @@ jobs:
1212

1313
services:
1414
db_service:
15-
image: kartoza/postgis:latest
15+
image: bitner/pgstac:0.1.7
1616
env:
1717
POSTGRES_USER: username
18-
POSTGRES_PASS: password
19-
POSTGRES_DBNAME: postgis
18+
POSTGRES_PASSWORD: password
19+
POSTGRES_DB: postgis
2020
POSTGRES_HOST: localhost
2121
POSTGRES_PORT: 5432
2222
ALLOW_IP_RANGE: 0.0.0.0/0
@@ -63,6 +63,10 @@ jobs:
6363
run: |
6464
pip install ./stac_fastapi/sqlalchemy[dev,server]
6565
66+
- name: Install pgstac stac-fastapi
67+
run: |
68+
pip install ./stac_fastapi/pgstac[dev,server]
69+
6670
- name: Run migration
6771
run: |
6872
cd stac_fastapi/sqlalchemy && alembic upgrade head
@@ -75,7 +79,19 @@ jobs:
7579

7680
- name: Run test suite
7781
run: |
78-
pipenv run pytest -svvv
82+
cd stac_fastapi/sqlalchemy && pipenv run pytest -svvv
83+
env:
84+
ENVIRONMENT: testing
85+
POSTGRES_USER: username
86+
POSTGRES_PASS: password
87+
POSTGRES_DBNAME: postgis
88+
POSTGRES_HOST_READER: localhost
89+
POSTGRES_HOST_WRITER: localhost
90+
POSTGRES_PORT: 5432
91+
92+
- name: Run test suite
93+
run: |
94+
cd stac_fastapi/pgstac && pipenv run pytest -svvv
7995
env:
8096
ENVIRONMENT: testing
8197
POSTGRES_USER: username

‎Dockerfile‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,5 @@ RUN mkdir -p /install && \
2020
pip install -e ./stac_fastapi/api[dev] && \
2121
pip install -e ./stac_fastapi/types[dev] && \
2222
pip install -e ./stac_fastapi/extensions[dev,tiles] && \
23-
pip install -e ./stac_fastapi/sqlalchemy[dev,server]
24-
25-
# FROM base
26-
# COPY --from=builder /install /usr/local
27-
28-
WORKDIR /app/stac_fastapi/sqlalchemy
29-
ENV APP_HOST=0.0.0.0
30-
ENV APP_PORT=80
31-
ENV RELOAD="true"
32-
33-
CMD if [ "$RELOAD" ]; then uvicorn stac_fastapi.sqlalchemy.app:app --host=${APP_HOST} --port=${APP_PORT} --reload ; \
34-
else gunicorn stac_fastapi.sqlalchemy.app:app --preload -k uvicorn.workers.UvicornWorker --bind ${APP_HOST}:${APP_PORT} --log-level info; fi
23+
pip install -e ./stac_fastapi/sqlalchemy[dev,server] && \
24+
pip install -e ./stac_fastapi/pgstac[dev,server]

‎Makefile‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@ run_docker = docker-compose run --rm \
66
-p ${EXTERNAL_APP_PORT}:${APP_PORT} \
77
-e APP_HOST=${APP_HOST} \
88
-e APP_PORT=${APP_PORT} \
9-
app
9+
app-sqlalchemy
1010

1111
.PHONY: image
1212
image:
13-
docker-compose build app
13+
docker-compose build
1414

1515
.PHONY: docker-run
1616
docker-run: image
@@ -20,6 +20,28 @@ docker-run: image
2020
docker-shell:
2121
$(run_docker) /bin/bash
2222

23+
.PHONY: test-sqlalchemy
24+
test-sqlalchemy:
25+
$(run_docker) /bin/bash -c 'export && cd /app/stac_fastapi/sqlalchemy/tests/ && pytest'
26+
27+
.PHONY: test-pgstac
28+
test-pgstac: pgstac-install
29+
$(run_docker) /bin/bash -c 'export && cd /app/stac_fastapi/pgstac/tests/ && pytest'
30+
2331
.PHONY: test
24-
test:
25-
$(run_docker) pytest
32+
test: test-sqlalchemy test-pgstac
33+
34+
.PHONY: pybase-install
35+
pybase-install:
36+
pip install wheel && \
37+
pip install -e ./stac_fastapi/api[dev] && \
38+
pip install -e ./stac_fastapi/types[dev] && \
39+
pip install -e ./stac_fastapi/extensions[dev,tiles]
40+
41+
.PHONY: pgstac-install
42+
pgstac-install: pybase-install
43+
pip install -e ./stac_fastapi/pgstac[dev,server]
44+
45+
.PHONY: sqlalchemy-install
46+
sqlalchemy-install: pybase-install
47+
pip install -e ./stac_fastapi/sqlalchemy[dev,server]

‎docker-compose.yml‎

Lines changed: 67 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
version: '3'
22

33
services:
4-
app:
5-
container_name: stac-fastapi
4+
app-sqlalchemy:
5+
container_name: stac-fastapi-sqlalchemy
66
image: stac-utils/stac-fastapi
77
build:
88
context: .
99
dockerfile: Dockerfile
1010
environment:
1111
- APP_HOST=0.0.0.0
1212
- APP_PORT=8081
13-
- RELOAD=true
13+
- RELOAD=false
1414
- ENVIRONMENT=local
1515
- POSTGRES_USER=username
1616
- POSTGRES_PASS=password
@@ -22,24 +22,81 @@ services:
2222
- VSI_CACHE=TRUE
2323
- GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES
2424
- GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
25+
- DB_MIN_CONN_SIZE=1
26+
- DB_MAX_CONN_SIZE=1
2527
ports:
2628
- "8081:8081"
2729
volumes:
28-
- ./:/app
30+
- ./stac_fastapi:/app/stac_fastapi
31+
- ./scripts:/app/scripts
2932
depends_on:
3033
- database
34+
command:
35+
python -m stac_fastapi.sqlalchemy.app
36+
37+
app-pgstac:
38+
container_name: stac-fastapi-pgstac
39+
image: stac-utils/stac-fastapi
40+
build:
41+
context: .
42+
dockerfile: Dockerfile
43+
environment:
44+
- APP_HOST=0.0.0.0
45+
- APP_PORT=8082
46+
- RELOAD=false
47+
- ENVIRONMENT=local
48+
- POSTGRES_USER=username
49+
- POSTGRES_PASS=password
50+
- POSTGRES_DBNAME=postgis
51+
- POSTGRES_HOST_READER=database
52+
- POSTGRES_HOST_WRITER=database
53+
- POSTGRES_PORT=5432
54+
- WEB_CONCURRENCY=10
55+
- VSI_CACHE=TRUE
56+
- GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES
57+
- GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
58+
- DB_MIN_CONN_SIZE=1
59+
- DB_MAX_CONN_SIZE=1
60+
ports:
61+
- "8082:8082"
62+
volumes:
63+
- ./stac_fastapi:/app/stac_fastapi
64+
- ./scripts:/app/scripts
65+
depends_on:
66+
- database
67+
command:
68+
python -m stac_fastapi.pgstac.app
3169

3270
database:
3371
container_name: stac-db
34-
image: postgis/postgis:12-3.0
72+
image: bitner/pgstac:0.2.4
3573
environment:
3674
- POSTGRES_USER=username
3775
- POSTGRES_PASSWORD=password
3876
- POSTGRES_DB=postgis
3977
ports:
4078
- "5439:5432"
79+
command: postgres -N 500
80+
81+
sqlalchemy-migration:
82+
image: stac-utils/stac-fastapi
83+
environment:
84+
- ENVIRONMENT=development
85+
- POSTGRES_USER=username
86+
- POSTGRES_PASS=password
87+
- POSTGRES_DBNAME=postgis
88+
- POSTGRES_HOST=database
89+
- POSTGRES_PORT=5432
90+
volumes:
91+
- ./stac_fastapi:/app/stac_fastapi
92+
- ./scripts:/app/scripts
93+
command: >
94+
bash -c "sleep 10 && cd stac_fastapi/sqlalchemy && alembic upgrade head && python /app/scripts/ingest_joplin.py http://app-sqlalchemy:8081"
95+
depends_on:
96+
- database
97+
- app-sqlalchemy
4198

42-
migration:
99+
pgstac-loadjoplin:
43100
image: stac-utils/stac-fastapi
44101
environment:
45102
- ENVIRONMENT=development
@@ -49,12 +106,13 @@ services:
49106
- POSTGRES_HOST=database
50107
- POSTGRES_PORT=5432
51108
volumes:
52-
- ./:/app
109+
- ./stac_fastapi:/app/stac_fastapi
110+
- ./scripts:/app/scripts
53111
command: >
54-
bash -c "sleep 10 && alembic upgrade head && python /app/scripts/ingest_joplin.py"
112+
bash -c "sleep 10 && python /app/scripts/ingest_joplin.py http://app-pgstac:8082"
55113
depends_on:
56114
- database
57-
- app
115+
- app-pgstac
58116

59117
networks:
60118
default:

‎scripts/ingest_joplin.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,21 @@
11
"""Ingest sample data during docker-compose"""
22
import json
3+
import sys
34
from pathlib import Path
45
from urllib.parse import urljoin
56

67
import requests
78

8-
bucket = "arturo-stac-api-test-data"
9-
app_host = "http://app:8081"
9+
workingdir = Path(__file__).parent.absolute()
10+
joplindata = workingdir.parent / "stac_fastapi" / "testdata" / "joplin"
1011

12+
app_host = sys.argv[1]
1113

12-
def ingest_joplin_data(data_dir=Path.cwd() / "tests" / "data" / "joplin"):
14+
if not app_host:
15+
raise Exception("You must include full path/port to stac instance")
16+
17+
18+
def ingest_joplin_data(app_host: str = app_host, data_dir: Path = joplindata):
1319
"""ingest data."""
1420

1521
with open(data_dir / "collection.json") as f:

‎stac_fastapi/api/stac_fastapi/api/app.py‎

Lines changed: 17 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from typing import Any, Dict, List, Optional, Type
33

44
import attr
5+
from brotli_asgi import BrotliMiddleware
56
from fastapi import APIRouter, FastAPI
67
from fastapi.openapi.utils import get_openapi
78
from stac_pydantic import Collection, Item, ItemCollection
@@ -16,10 +17,7 @@
1617
SearchGetRequest,
1718
_create_request_model,
1819
)
19-
from stac_fastapi.api.routes import (
20-
create_endpoint_from_model,
21-
create_endpoint_with_depends,
22-
)
20+
from stac_fastapi.api.routes import create_endpoint
2321

2422
# TODO: make this module not depend on `stac_fastapi.extensions`
2523
from stac_fastapi.extensions.core import FieldsExtension
@@ -60,6 +58,7 @@ class StacApi:
6058
default=attr.Factory(lambda: DEFAULT_STATUS_CODES)
6159
)
6260
app: FastAPI = attr.ib(default=attr.Factory(FastAPI))
61+
search_request_model = attr.ib(default=STACSearch)
6362
title: str = attr.ib(default="Arturo STAC API")
6463
version: str = attr.ib(default="0.1")
6564

@@ -94,7 +93,8 @@ def register_core(self):
9493
Returns:
9594
None
9695
"""
97-
search_request_model = _create_request_model(STACSearch)
96+
search_request_model = _create_request_model(self.search_request_model)
97+
9898
fields_ext = self.get_extension(FieldsExtension)
9999
router = APIRouter()
100100
router.add_api_route(
@@ -104,9 +104,7 @@ def register_core(self):
104104
response_model_exclude_unset=False,
105105
response_model_exclude_none=True,
106106
methods=["GET"],
107-
endpoint=create_endpoint_with_depends(
108-
self.client.landing_page, EmptyRequest
109-
),
107+
endpoint=create_endpoint(self.client.landing_page, EmptyRequest),
110108
)
111109
router.add_api_route(
112110
name="Conformance Classes",
@@ -115,9 +113,7 @@ def register_core(self):
115113
response_model_exclude_unset=True,
116114
response_model_exclude_none=True,
117115
methods=["GET"],
118-
endpoint=create_endpoint_with_depends(
119-
self.client.conformance, EmptyRequest
120-
),
116+
endpoint=create_endpoint(self.client.conformance, EmptyRequest),
121117
)
122118
router.add_api_route(
123119
name="Get Item",
@@ -126,7 +122,7 @@ def register_core(self):
126122
response_model_exclude_unset=True,
127123
response_model_exclude_none=True,
128124
methods=["GET"],
129-
endpoint=create_endpoint_with_depends(self.client.get_item, ItemUri),
125+
endpoint=create_endpoint(self.client.get_item, ItemUri),
130126
)
131127
router.add_api_route(
132128
name="Search",
@@ -135,9 +131,7 @@ def register_core(self):
135131
response_model_exclude_unset=True,
136132
response_model_exclude_none=True,
137133
methods=["POST"],
138-
endpoint=create_endpoint_from_model(
139-
self.client.post_search, search_request_model
140-
),
134+
endpoint=create_endpoint(self.client.post_search, search_request_model),
141135
),
142136
router.add_api_route(
143137
name="Search",
@@ -146,9 +140,7 @@ def register_core(self):
146140
response_model_exclude_unset=True,
147141
response_model_exclude_none=True,
148142
methods=["GET"],
149-
endpoint=create_endpoint_with_depends(
150-
self.client.get_search, SearchGetRequest
151-
),
143+
endpoint=create_endpoint(self.client.get_search, SearchGetRequest),
152144
)
153145
router.add_api_route(
154146
name="Get Collections",
@@ -157,9 +149,7 @@ def register_core(self):
157149
response_model_exclude_unset=True,
158150
response_model_exclude_none=True,
159151
methods=["GET"],
160-
endpoint=create_endpoint_with_depends(
161-
self.client.all_collections, EmptyRequest
162-
),
152+
endpoint=create_endpoint(self.client.all_collections, EmptyRequest),
163153
)
164154
router.add_api_route(
165155
name="Get Collection",
@@ -168,9 +158,7 @@ def register_core(self):
168158
response_model_exclude_unset=True,
169159
response_model_exclude_none=True,
170160
methods=["GET"],
171-
endpoint=create_endpoint_with_depends(
172-
self.client.get_collection, CollectionUri
173-
),
161+
endpoint=create_endpoint(self.client.get_collection, CollectionUri),
174162
)
175163
router.add_api_route(
176164
name="Get ItemCollection",
@@ -179,9 +167,7 @@ def register_core(self):
179167
response_model_exclude_unset=True,
180168
response_model_exclude_none=True,
181169
methods=["GET"],
182-
endpoint=create_endpoint_with_depends(
183-
self.client.item_collection, ItemCollectionUri
184-
),
170+
endpoint=create_endpoint(self.client.item_collection, ItemCollectionUri),
185171
)
186172
self.app.include_router(router)
187173

@@ -224,6 +210,7 @@ def __attrs_post_init__(self):
224210
self.settings.default_includes = fields_ext.default_includes
225211

226212
Settings.set(self.settings)
213+
self.app.state.settings = self.settings
227214

228215
self.register_core()
229216
# register extensions
@@ -238,3 +225,6 @@ def __attrs_post_init__(self):
238225

239226
# customize openapi
240227
self.app.openapi = self.customize_openapi
228+
229+
# add compression middleware
230+
self.app.add_middleware(BrotliMiddleware)

0 commit comments

Comments
 (0)