[codex] add discovery headers (#129)

* add discovery headers

* split discovery link headers

Author: basit3407

static/_headers

@@ -13,6 +13,23 @@
   # Cache-Control for HTML pages: no long-term cache
   Cache-Control: public, max-age=0, must-revalidate
 
+# Homepage discovery links for agents and API consumers
+/
+  Link: </llms.txt>; rel="describedby"; type="text/plain"
+  Link: </openAPI/content/v4.json>; rel="service-desc"; type="application/json"
+  Link: </openAPI/oauth2-apis/v1.json>; rel="service-desc"; type="application/json"
+  Link: </openAPI/search/v1.json>; rel="service-desc"; type="application/json"
+  Link: </openAPI/user-related-apis/v1.json>; rel="service-desc"; type="application/json"
+  Link: </docs/content_apis_versioned/content-apis/>; rel="service-doc"; type="text/html"
+  Link: </docs/oauth2_apis_versioned/oauth-2-apis/>; rel="service-doc"; type="text/html"
+  Link: </docs/search_apis_versioned/quran-foundation-search-api/>; rel="service-doc"; type="text/html"
+  Link: </docs/user_related_apis_versioned/user-related-apis/>; rel="service-doc"; type="text/html"
+  Link: </docs/tutorials/oidc/getting-started-with-oauth2/>; rel="service-doc"; type="text/html"
+  Link: </docs/quickstart/>; rel="service-doc"; type="text/html"
+  Link: </docs/tutorials/oidc/user-apis-quickstart/>; rel="service-doc"; type="text/html"
+  Link: </docs/user_related_apis_versioned/scopes/>; rel="service-doc"; type="text/html"
+  Link: </request-access/>; rel="service-doc"; type="text/html"
+
 # Cache hashed JavaScript and CSS files for 1 year (they have content hash in filename)
 /assets/js/*.js
   Cache-Control: public, max-age=31536000, immutable
@@ -41,6 +58,10 @@
 /sitemap.xml
   Cache-Control: public, max-age=86400
 
+/robots.txt
+  ! Cache-Control
+  Cache-Control: public, max-age=86400
+
 /opensearch.xml
   Cache-Control: public, max-age=86400