Return existing links in pgstac (#282)

* Add venv to ignore files

* Add run-joplin-pgstac make command

* Modify ingest logic to update if exists

* Add license to collections; test license link

* Fix bug where existing links were not returned

* Update CHANGES

* Use get_links instead of new _extended_links

Explain why we resolve the href for extra_links in get_links

* Test item extra_links with relative href

Author: lossyrob

.dockerignore

@@ -10,3 +10,5 @@ coverage.xml
 *.log
 .git
 .envrc
+
+venv

.gitignore

@@ -126,3 +126,6 @@ docs/api/*
 
 # Direnv
 .envrc
+
+# Virtualenv
+venv

CHANGES.md

@@ -10,6 +10,8 @@
 
 ### Fixed
 
+* Links stored with Collections and Items (e.g. license links) are now returned with those STAC objects ([#282](https://github.com/stac-utils/stac-fastapi/pull/282))
+
 ## [2.2.0]
 
 ### Added

Makefile

@@ -50,6 +50,10 @@ run-database:
 run-joplin-sqlalchemy:
 	docker-compose run --rm loadjoplin-sqlalchemy
 
+.PHONY: run-joplin-pgstac
+run-joplin-pgstac:
+	docker-compose run --rm loadjoplin-pgstac
+
 .PHONY: test
 test: test-sqlalchemy test-pgstac
 

scripts/ingest_joplin.py

@@ -15,27 +15,33 @@
     raise Exception("You must include full path/port to stac instance")
 
 
+def post_or_put(url: str, data: dict):
+    """Post or put data to url."""
+    r = requests.post(url, json=data)
+    if r.status_code == 409:
+        # Exists, so update
+        r = requests.put(url, json=data)
+        # Unchanged may throw a 404
+        if not r.status_code == 404:
+            r.raise_for_status()
+    else:
+        r.raise_for_status()
+
+
 def ingest_joplin_data(app_host: str = app_host, data_dir: Path = joplindata):
     """ingest data."""
 
     with open(data_dir / "collection.json") as f:
         collection = json.load(f)
 
-    r = requests.post(urljoin(app_host, "collections"), json=collection)
-    if r.status_code not in (200, 409):
-        r.raise_for_status()
+    post_or_put(urljoin(app_host, "/collections"), collection)
 
     with open(data_dir / "index.geojson") as f:
         index = json.load(f)
 
     for feat in index["features"]:
         del feat["stac_extensions"]
-        r = requests.post(
-            urljoin(app_host, f"collections/{collection['id']}/items"), json=feat
-        )
-        if r.status_code == 409:
-            continue
-        r.raise_for_status()
+        post_or_put(urljoin(app_host, f"collections/{collection['id']}/items"), feat)
 
 
 if __name__ == "__main__":

stac_fastapi/pgstac/stac_fastapi/pgstac/core.py

@@ -47,8 +47,10 @@ async def all_collections(self, **kwargs) -> Collections:
                 coll = Collection(**c)
                 coll["links"] = await CollectionLinks(
                     collection_id=coll["id"], request=request
-                ).get_links()
+                ).get_links(extra_links=coll.get("links"))
+
                 linked_collections.append(coll)
+
         links = [
             {
                 "rel": Relations.root.value,
@@ -94,8 +96,11 @@ async def get_collection(self, id: str, **kwargs) -> Collection:
             collection = await conn.fetchval(q, *p)
         if collection is None:
             raise NotFoundError(f"Collection {id} does not exist.")
-        links = await CollectionLinks(collection_id=id, request=request).get_links()
-        collection["links"] = links
+
+        collection["links"] = await CollectionLinks(
+            collection_id=id, request=request
+        ).get_links(extra_links=collection.get("links"))
+
         return Collection(**collection)
 
     async def _search_base(
@@ -147,12 +152,12 @@ async def _search_base(
                 # TODO: feature.collection is not always included
                 # This code fails if it's left outside of the fields expression
                 # I've fields extension updated test cases to always include feature.collection
-                links = await ItemLinks(
+                feature["links"] = await ItemLinks(
                     collection_id=feature["collection"],
                     item_id=feature["id"],
                     request=request,
-                ).get_links()
-                feature["links"] = links
+                ).get_links(extra_links=feature.get("links"))
+
                 exclude = search_request.fields.exclude
                 if exclude and len(exclude) == 0:
                     exclude = None

stac_fastapi/pgstac/stac_fastapi/pgstac/models/links.py

@@ -66,7 +66,7 @@ def link_root(self) -> Dict:
             rel=Relations.root.value, type=MimeTypes.json.value, href=self.base_url
         )
 
-    def create_links(self) -> List[Dict]:
+    def create_links(self) -> List[Dict[str, Any]]:
         """Return all inferred links."""
         links = []
         for name in dir(self):
@@ -77,7 +77,7 @@ def create_links(self) -> List[Dict]:
         return links
 
     async def get_links(
-        self, extra_links: List[Dict[str, Any]] = []
+        self, extra_links: Optional[List[Dict[str, Any]]] = None
     ) -> List[Dict[str, Any]]:
         """
         Generate all the links.
@@ -91,11 +91,22 @@ async def get_links(
         # join passed in links with generated links
         # and update relative paths
         links = self.create_links()
-        if extra_links is not None and len(extra_links) >= 1:
-            for link in extra_links:
-                if link["rel"] not in INFERRED_LINK_RELS:
-                    link["href"] = self.resolve(link["href"])
-                    links.append(link)
+
+        if extra_links:
+            # For extra links passed in,
+            # add links modified with a resolved href.
+            # Drop any links that are dynamically
+            # determined by the server (e.g. self, parent, etc.)
+            # Resolving the href allows for relative paths
+            # to be stored in pgstac and for the hrefs in the
+            # links of response STAC objects to be resolved
+            # to the request url.
+            links += [
+                {**link, "href": self.resolve(link["href"])}
+                for link in extra_links
+                if link["rel"] not in INFERRED_LINK_RELS
+            ]
+
         return links
 
 

stac_fastapi/pgstac/tests/data/test_collection.json

@@ -100,24 +100,9 @@
   },
   "links": [
     {
-      "href": "http://localhost:8081/collections/landsat-8-l1",
-      "rel": "self",
-      "type": "application/json"
-    },
-    {
-      "href": "http://localhost:8081/",
-      "rel": "parent",
-      "type": "application/json"
-    },
-    {
-      "href": "http://localhost:8081/collections/landsat-8-l1/items",
-      "rel": "item",
-      "type": "application/geo+json"
-    },
-    {
-      "href": "http://localhost:8081/",
-      "rel": "root",
-      "type": "application/json"
+        "rel": "license",
+        "href": "https://creativecommons.org/licenses/publicdomain/",
+        "title": "public domain"
     }
   ],
   "title": "Landsat 8 L1",

stac_fastapi/pgstac/tests/data/test_item.json

@@ -500,6 +500,11 @@
       "href": "http://localhost:8081/",
       "rel": "root",
       "type": "application/json"
+    },
+    {
+      "href": "preview.html",
+      "rel": "preview",
+      "type": "application/html"
     }
   ]
 }

stac_fastapi/pgstac/tests/resources/test_collection.py

@@ -164,3 +164,14 @@ async def test_returns_valid_links_in_collections(app_client, load_test_data):
         for i in collection.to_dict()["links"]
         if i not in single_coll_mocked_link.to_dict()["links"]
     ] == []
+
+
+@pytest.mark.asyncio
+async def test_returns_license_link(app_client, load_test_collection):
+    coll = load_test_collection
+
+    resp = await app_client.get(f"/collections/{coll.id}")
+    assert resp.status_code == 200
+    resp_json = resp.json()
+    link_rel_types = [link["rel"] for link in resp_json["links"]]
+    assert "license" in link_rel_types