From 6b25e56e5e5e322e366b833f7971e51ac59d621a Mon Sep 17 00:00:00 2001 From: Kamil Monicz Date: Sat, 12 Apr 2025 07:34:49 +0200 Subject: [PATCH 1/8] Zero-config dynamically-generated queryables, Performance fixes (#351) **Related Issue(s):** - Fixes https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/345 - Fixes https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/344 - Fixes https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/336 **Description:** This PR consists of self-contained commits (except the first commit that provides database_logic deduplication), making it easy to change or remove individual patches. It addresses several small issues, improves the performance of certain methods, and adds support for dynamically-generated queryables. This enhancement doesn't require any new configuration as queryables are generated on the fly based on the os/es mappings. The implementation is designed for extensibility, with built-in logic for augmenting fields metadata with additional information. Currently, it only includes the _DEFAULT_QUERYABLES configuration, which was simply copied from the pre-PR code. Example queryables response: ```json {"$schema":"https://json-schema.org/draft/2019-09/schema","$id":"https://stac-api.example.com/queryables","type":"object","title":"Queryables for STAC API","description":"Queryable names for the STAC API Item Search filter.","properties":{"bbox":{"title":"Bbox","type":"number"},"collection":{"description":"Collection","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection","title":"Collection","type":"string"},"geometry":{"description":"Geometry","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry","title":"Geometry","type":"object"},"id":{"description":"ID","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id","title":"Id","type":"string"},"stac_extensions":{"title":"Stac Extensions","type":"string"},"stac_version":{"title":"Stac Version","type":"string"},"type":{"title":"Type","type":"string"},"constellation":{"title":"Constellation","type":"string"},"created":{"description":"Creation Timestamp","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created","title":"Created","type":"string","format":"date-time"},"datetime":{"description":"Acquisition Timestamp","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime","title":"Datetime","type":"string","format":"date-time"},"end_datetime":{"title":"End Datetime","type":"string","format":"date-time"},"eopf:datatake_id":{"title":"Eopf:Datatake Id","type":"string"},"eopf:instrument_configuration_id":{"title":"Eopf:Instrument Configuration Id","type":"number"},"instruments":{"title":"Instruments","type":"string"},"platform":{"title":"Platform","type":"string"},"processing:datetime":{"title":"Processing:Datetime","type":"string","format":"date-time"},"processing:facility":{"title":"Processing:Facility","type":"string"},"processing:level":{"title":"Processing:Level","type":"string"},"processing:version":{"title":"Processing:Version","type":"string"},"product:timeliness":{"title":"Product:Timeliness","type":"string"},"product:timeliness_category":{"title":"Product:Timeliness Category","type":"string"},"product:type":{"title":"Product:Type","type":"string"},"published":{"title":"Published","type":"string","format":"date-time"},"sar:center_frequency":{"title":"Sar:Center Frequency","type":"number"},"sar:frequency_band":{"title":"Sar:Frequency Band","type":"string"},"sar:instrument_mode":{"title":"Sar:Instrument Mode","type":"string"},"sar:observation_direction":{"title":"Sar:Observation Direction","type":"string"},"sar:pixel_spacing_azimuth":{"title":"Sar:Pixel Spacing Azimuth","type":"number"},"sar:pixel_spacing_range":{"title":"Sar:Pixel Spacing Range","type":"number"},"sar:polarizations":{"title":"Sar:Polarizations","type":"string"},"sar:resolution_azimuth":{"title":"Sar:Resolution Azimuth","type":"number"},"sar:resolution_range":{"title":"Sar:Resolution Range","type":"number"},"sat:absolute_orbit":{"title":"Sat:Absolute Orbit","type":"integer"},"sat:orbit_cycle":{"title":"Sat:Orbit Cycle","type":"number"},"sat:orbit_state":{"title":"Sat:Orbit State","type":"string"},"sat:platform_international_designator":{"title":"Sat:Platform International Designator","type":"string"},"sat:relative_orbit":{"title":"Sat:Relative Orbit","type":"integer"},"start_datetime":{"title":"Start Datetime","type":"string","format":"date-time"},"updated":{"description":"Creation Timestamp","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated","title":"Updated","type":"string","format":"date-time"},"view:azimuth":{"title":"View:Azimuth","type":"number"},"view:incidence_angle":{"title":"View:Incidence Angle","type":"number"},"auth:schemes.oidc.openIdConnectUrl":{"title":"Auth:Schemes.Oidc.Openidconnecturl","type":"string"},"auth:schemes.oidc.type":{"title":"Auth:Schemes.Oidc.Type","type":"string"},"auth:schemes.s3.type":{"title":"Auth:Schemes.S3.Type","type":"string"},"storage:schemes.cdse-s3.description":{"title":"Storage:Schemes.Cdse-S3.Description","type":"string"},"storage:schemes.cdse-s3.platform":{"title":"Storage:Schemes.Cdse-S3.Platform","type":"string"},"storage:schemes.cdse-s3.requester_pays":{"title":"Storage:Schemes.Cdse-S3.Requester Pays","type":"boolean"},"storage:schemes.cdse-s3.title":{"title":"Storage:Schemes.Cdse-S3.Title","type":"string"},"storage:schemes.cdse-s3.type":{"title":"Storage:Schemes.Cdse-S3.Type","type":"string"},"storage:schemes.creodias-s3.description":{"title":"Storage:Schemes.Creodias-S3.Description","type":"string"},"storage:schemes.creodias-s3.platform":{"title":"Storage:Schemes.Creodias-S3.Platform","type":"string"},"storage:schemes.creodias-s3.requester_pays":{"title":"Storage:Schemes.Creodias-S3.Requester Pays","type":"boolean"},"storage:schemes.creodias-s3.title":{"title":"Storage:Schemes.Creodias-S3.Title","type":"string"},"storage:schemes.creodias-s3.type":{"title":"Storage:Schemes.Creodias-S3.Type","type":"string"}},"additionalProperties":false} ``` PS. I think the auto-generated "title" should be removed completely, but I included it because I found it to be common practice in some STAC projects. I'm not sure how you feel about it. **PR Checklist:** - [ ] Code is formatted and linted (run `pre-commit run --all-files`) - [ ] Tests pass (run `make test`) - [ ] Documentation has been updated to reflect changes, if applicable - [ ] Changes are added to the changelog --------- Co-authored-by: Jonathan Healy --- CHANGELOG.md | 7 + stac_fastapi/core/setup.py | 2 +- stac_fastapi/core/stac_fastapi/core/core.py | 178 +++++++++---- .../core/stac_fastapi/core/database_logic.py | 226 ++++++++++++++++ .../stac_fastapi/core/extensions/query.py | 4 +- .../core/stac_fastapi/core/models/links.py | 2 +- .../stac_fastapi/elasticsearch/app.py | 8 +- .../elasticsearch/database_logic.py | 247 +++--------------- .../opensearch/stac_fastapi/opensearch/app.py | 8 +- .../stac_fastapi/opensearch/database_logic.py | 247 +++--------------- .../tests/rate_limit/test_rate_limit.py | 4 +- 11 files changed, 447 insertions(+), 486 deletions(-) create mode 100644 stac_fastapi/core/stac_fastapi/core/database_logic.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d58271e..04b4d793 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- Added support for dynamically-generated queryables based on Elasticsearch/OpenSearch mappings, with extensible metadata augmentation [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) +- Included default queryables configuration for seamless integration. [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) ### Changed +- Refactored database logic to reduce duplication [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) +- Replaced `fastapi-slim` with `fastapi` dependency [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) + +### Fixed +- Improved performance of `mk_actions` and `filter-links` methods [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) ## [v3.2.5] - 2025-04-07 diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 01191c1b..aedbe231 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "fastapi-slim", + "fastapi", "attrs>=23.2.0", "pydantic", "stac_pydantic>=3", diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 56afcbc8..11bd34b4 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -1,10 +1,11 @@ """Core client.""" import logging +from collections import deque from datetime import datetime as datetime_type from datetime import timezone from enum import Enum -from typing import Any, Dict, List, Optional, Set, Type, Union +from typing import Any, Dict, List, Literal, Optional, Set, Type, Union from urllib.parse import unquote_plus, urljoin import attr @@ -41,8 +42,6 @@ logger = logging.getLogger(__name__) -NumType = Union[float, int] - @attr.s class CoreClient(AsyncBaseCoreClient): @@ -907,11 +906,81 @@ def bulk_item_insert( return f"Successfully added {len(processed_items)} Items." +_DEFAULT_QUERYABLES: Dict[str, Dict[str, Any]] = { + "id": { + "description": "ID", + "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id", + }, + "collection": { + "description": "Collection", + "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection", + }, + "geometry": { + "description": "Geometry", + "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry", + }, + "datetime": { + "description": "Acquisition Timestamp", + "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime", + }, + "created": { + "description": "Creation Timestamp", + "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created", + }, + "updated": { + "description": "Creation Timestamp", + "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated", + }, + "cloud_cover": { + "description": "Cloud Cover", + "$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover", + }, + "cloud_shadow_percentage": { + "title": "Cloud Shadow Percentage", + "description": "Cloud Shadow Percentage", + "type": "number", + "minimum": 0, + "maximum": 100, + }, + "nodata_pixel_percentage": { + "title": "No Data Pixel Percentage", + "description": "No Data Pixel Percentage", + "type": "number", + "minimum": 0, + "maximum": 100, + }, +} + +_ES_MAPPING_TYPE_TO_JSON: Dict[ + str, Literal["string", "number", "boolean", "object", "array", "null"] +] = { + "date": "string", + "date_nanos": "string", + "keyword": "string", + "match_only_text": "string", + "text": "string", + "wildcard": "string", + "byte": "number", + "double": "number", + "float": "number", + "half_float": "number", + "long": "number", + "scaled_float": "number", + "short": "number", + "token_count": "number", + "unsigned_long": "number", + "geo_point": "object", + "geo_shape": "object", + "nested": "array", +} + + @attr.s class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient): """Defines a pattern for implementing the STAC filter extension.""" - # todo: use the ES _mapping endpoint to dynamically find what fields exist + database: BaseDatabaseLogic = attr.ib() + async def get_queryables( self, collection_id: Optional[str] = None, **kwargs ) -> Dict[str, Any]: @@ -932,55 +1001,62 @@ async def get_queryables( Returns: Dict[str, Any]: A dictionary containing the queryables for the given collection. """ - return { + queryables: Dict[str, Any] = { "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://stac-api.example.com/queryables", "type": "object", - "title": "Queryables for Example STAC API", - "description": "Queryable names for the example STAC API Item Search filter.", - "properties": { - "id": { - "description": "ID", - "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id", - }, - "collection": { - "description": "Collection", - "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection", - }, - "geometry": { - "description": "Geometry", - "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry", - }, - "datetime": { - "description": "Acquisition Timestamp", - "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime", - }, - "created": { - "description": "Creation Timestamp", - "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created", - }, - "updated": { - "description": "Creation Timestamp", - "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated", - }, - "cloud_cover": { - "description": "Cloud Cover", - "$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover", - }, - "cloud_shadow_percentage": { - "description": "Cloud Shadow Percentage", - "title": "Cloud Shadow Percentage", - "type": "number", - "minimum": 0, - "maximum": 100, - }, - "nodata_pixel_percentage": { - "description": "No Data Pixel Percentage", - "title": "No Data Pixel Percentage", - "type": "number", - "minimum": 0, - "maximum": 100, - }, - }, + "title": "Queryables for STAC API", + "description": "Queryable names for the STAC API Item Search filter.", + "properties": _DEFAULT_QUERYABLES, "additionalProperties": True, } + if not collection_id: + return queryables + + properties: Dict[str, Any] = queryables["properties"] + queryables.update( + { + "properties": properties, + "additionalProperties": False, + } + ) + + mapping_data = await self.database.get_items_mapping(collection_id) + mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"] + stack = deque(mapping_properties.items()) + + while stack: + field_name, field_def = stack.popleft() + + # Iterate over nested fields + field_properties = field_def.get("properties") + if field_properties: + # Fields in Item Properties should be exposed with their un-prefixed names, + # and not require expressions to prefix them with properties, + # e.g., eo:cloud_cover instead of properties.eo:cloud_cover. + if field_name == "properties": + stack.extend(field_properties.items()) + else: + stack.extend( + (f"{field_name}.{k}", v) for k, v in field_properties.items() + ) + + # Skip non-indexed or disabled fields + field_type = field_def.get("type") + if not field_type or not field_def.get("enabled", True): + continue + + # Generate field properties + field_result = _DEFAULT_QUERYABLES.get(field_name, {}) + properties[field_name] = field_result + + field_name_human = field_name.replace("_", " ").title() + field_result.setdefault("title", field_name_human) + + field_type_json = _ES_MAPPING_TYPE_TO_JSON.get(field_type, field_type) + field_result.setdefault("type", field_type_json) + + if field_type in {"date", "date_nanos"}: + field_result.setdefault("format", "date-time") + + return queryables diff --git a/stac_fastapi/core/stac_fastapi/core/database_logic.py b/stac_fastapi/core/stac_fastapi/core/database_logic.py new file mode 100644 index 00000000..7ddd8af7 --- /dev/null +++ b/stac_fastapi/core/stac_fastapi/core/database_logic.py @@ -0,0 +1,226 @@ +"""Database logic core.""" + +import os +from functools import lru_cache +from typing import Any, Dict, List, Optional, Protocol + +from stac_fastapi.types.stac import Item + + +# stac_pydantic classes extend _GeometryBase, which doesn't have a type field, +# So create our own Protocol for typing +# Union[ Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon, GeometryCollection] +class Geometry(Protocol): # noqa + type: str + coordinates: Any + + +COLLECTIONS_INDEX = os.getenv("STAC_COLLECTIONS_INDEX", "collections") +ITEMS_INDEX_PREFIX = os.getenv("STAC_ITEMS_INDEX_PREFIX", "items_") + +ES_INDEX_NAME_UNSUPPORTED_CHARS = { + "\\", + "/", + "*", + "?", + '"', + "<", + ">", + "|", + " ", + ",", + "#", + ":", +} + +_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE = str.maketrans( + "", "", "".join(ES_INDEX_NAME_UNSUPPORTED_CHARS) +) + +ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}*" + +DEFAULT_SORT = { + "properties.datetime": {"order": "desc"}, + "id": {"order": "desc"}, + "collection": {"order": "desc"}, +} + +ES_ITEMS_SETTINGS = { + "index": { + "sort.field": list(DEFAULT_SORT.keys()), + "sort.order": [v["order"] for v in DEFAULT_SORT.values()], + } +} + +ES_MAPPINGS_DYNAMIC_TEMPLATES = [ + # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md + { + "descriptions": { + "match_mapping_type": "string", + "match": "description", + "mapping": {"type": "text"}, + } + }, + { + "titles": { + "match_mapping_type": "string", + "match": "title", + "mapping": {"type": "text"}, + } + }, + # Projection Extension https://github.com/stac-extensions/projection + {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}}, + { + "proj_projjson": { + "match": "proj:projjson", + "mapping": {"type": "object", "enabled": False}, + } + }, + { + "proj_centroid": { + "match": "proj:centroid", + "mapping": {"type": "geo_point"}, + } + }, + { + "proj_geometry": { + "match": "proj:geometry", + "mapping": {"type": "object", "enabled": False}, + } + }, + { + "no_index_href": { + "match": "href", + "mapping": {"type": "text", "index": False}, + } + }, + # Default all other strings not otherwise specified to keyword + {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, + {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, +] + +ES_ITEMS_MAPPINGS = { + "numeric_detection": False, + "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, + "properties": { + "id": {"type": "keyword"}, + "collection": {"type": "keyword"}, + "geometry": {"type": "geo_shape"}, + "assets": {"type": "object", "enabled": False}, + "links": {"type": "object", "enabled": False}, + "properties": { + "type": "object", + "properties": { + # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md + "datetime": {"type": "date"}, + "start_datetime": {"type": "date"}, + "end_datetime": {"type": "date"}, + "created": {"type": "date"}, + "updated": {"type": "date"}, + # Satellite Extension https://github.com/stac-extensions/sat + "sat:absolute_orbit": {"type": "integer"}, + "sat:relative_orbit": {"type": "integer"}, + }, + }, + }, +} + +ES_COLLECTIONS_MAPPINGS = { + "numeric_detection": False, + "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, + "properties": { + "id": {"type": "keyword"}, + "extent.spatial.bbox": {"type": "long"}, + "extent.temporal.interval": {"type": "date"}, + "providers": {"type": "object", "enabled": False}, + "links": {"type": "object", "enabled": False}, + "item_assets": {"type": "object", "enabled": False}, + }, +} + + +@lru_cache(256) +def index_by_collection_id(collection_id: str) -> str: + """ + Translate a collection id into an Elasticsearch index name. + + Args: + collection_id (str): The collection id to translate into an index name. + + Returns: + str: The index name derived from the collection id. + """ + cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE) + return ( + f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{collection_id.encode('utf-8').hex()}" + ) + + +@lru_cache(256) +def index_alias_by_collection_id(collection_id: str) -> str: + """ + Translate a collection id into an Elasticsearch index alias. + + Args: + collection_id (str): The collection id to translate into an index alias. + + Returns: + str: The index alias derived from the collection id. + """ + cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE) + return f"{ITEMS_INDEX_PREFIX}{cleaned}" + + +def indices(collection_ids: Optional[List[str]]) -> str: + """ + Get a comma-separated string of index names for a given list of collection ids. + + Args: + collection_ids: A list of collection ids. + + Returns: + A string of comma-separated index names. If `collection_ids` is empty, returns the default indices. + """ + return ( + ",".join(map(index_alias_by_collection_id, collection_ids)) + if collection_ids + else ITEM_INDICES + ) + + +def mk_item_id(item_id: str, collection_id: str) -> str: + """Create the document id for an Item in Elasticsearch. + + Args: + item_id (str): The id of the Item. + collection_id (str): The id of the Collection that the Item belongs to. + + Returns: + str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character. + """ + return f"{item_id}|{collection_id}" + + +def mk_actions(collection_id: str, processed_items: List[Item]) -> List[Dict[str, Any]]: + """Create Elasticsearch bulk actions for a list of processed items. + + Args: + collection_id (str): The identifier for the collection the items belong to. + processed_items (List[Item]): The list of processed items to be bulk indexed. + + Returns: + List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed, + each action being a dictionary with the following keys: + - `_index`: the index to store the document in. + - `_id`: the document's identifier. + - `_source`: the source of the document. + """ + index_alias = index_alias_by_collection_id(collection_id) + return [ + { + "_index": index_alias, + "_id": mk_item_id(item["id"], item["collection"]), + "_source": item, + } + for item in processed_items + ] diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/query.py b/stac_fastapi/core/stac_fastapi/core/extensions/query.py index 97342c66..3084cbf8 100644 --- a/stac_fastapi/core/stac_fastapi/core/extensions/query.py +++ b/stac_fastapi/core/stac_fastapi/core/extensions/query.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from enum import auto from types import DynamicClassAttribute -from typing import Any, Callable, Dict, Optional, Union +from typing import Any, Callable, Dict, Optional from pydantic import BaseModel, root_validator from stac_pydantic.utils import AutoValueEnum @@ -17,8 +17,6 @@ logger = logging.getLogger("uvicorn") logger.setLevel(logging.INFO) -# Be careful: https://github.com/samuelcolvin/pydantic/issues/1423#issuecomment-642797287 -NumType = Union[float, int] class Operator(str, AutoValueEnum): diff --git a/stac_fastapi/core/stac_fastapi/core/models/links.py b/stac_fastapi/core/stac_fastapi/core/models/links.py index 76f0ce5b..f72d4ed4 100644 --- a/stac_fastapi/core/stac_fastapi/core/models/links.py +++ b/stac_fastapi/core/stac_fastapi/core/models/links.py @@ -12,7 +12,7 @@ # These can be inferred from the item/collection, so they aren't included in the database # Instead they are dynamically generated when querying the database using the classes defined below -INFERRED_LINK_RELS = ["self", "item", "parent", "collection", "root"] +INFERRED_LINK_RELS = {"self", "item", "parent", "collection", "root"} def merge_params(url: str, newparams: Dict) -> str: diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 5e6307e7..9510eaa6 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -39,13 +39,15 @@ settings = ElasticsearchSettings() session = Session.create_from_settings(settings) -filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient()) +database_logic = DatabaseLogic() + +filter_extension = FilterExtension( + client=EsAsyncBaseFiltersClient(database=database_logic) +) filter_extension.conformance_classes.append( "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" ) -database_logic = DatabaseLogic() - aggregation_extension = AggregationExtension( client=EsAsyncAggregationClient( database=database_logic, session=session, settings=settings diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 0f272218..c46b208d 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -3,16 +3,30 @@ import asyncio import json import logging -import os from base64 import urlsafe_b64decode, urlsafe_b64encode from copy import deepcopy -from typing import Any, Dict, Iterable, List, Optional, Protocol, Tuple, Type, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Type import attr from elasticsearch_dsl import Q, Search from starlette.requests import Request from elasticsearch import exceptions, helpers # type: ignore +from stac_fastapi.core.database_logic import ( + COLLECTIONS_INDEX, + DEFAULT_SORT, + ES_COLLECTIONS_MAPPINGS, + ES_ITEMS_MAPPINGS, + ES_ITEMS_SETTINGS, + ITEM_INDICES, + ITEMS_INDEX_PREFIX, + Geometry, + index_alias_by_collection_id, + index_by_collection_id, + indices, + mk_actions, + mk_item_id, +) from stac_fastapi.core.extensions import filter from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon @@ -25,168 +39,6 @@ logger = logging.getLogger(__name__) -NumType = Union[float, int] - -COLLECTIONS_INDEX = os.getenv("STAC_COLLECTIONS_INDEX", "collections") -ITEMS_INDEX_PREFIX = os.getenv("STAC_ITEMS_INDEX_PREFIX", "items_") -ES_INDEX_NAME_UNSUPPORTED_CHARS = { - "\\", - "/", - "*", - "?", - '"', - "<", - ">", - "|", - " ", - ",", - "#", - ":", -} - -ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}*" - -DEFAULT_SORT = { - "properties.datetime": {"order": "desc"}, - "id": {"order": "desc"}, - "collection": {"order": "desc"}, -} - -ES_ITEMS_SETTINGS = { - "index": { - "sort.field": list(DEFAULT_SORT.keys()), - "sort.order": [v["order"] for v in DEFAULT_SORT.values()], - } -} - -ES_MAPPINGS_DYNAMIC_TEMPLATES = [ - # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md - { - "descriptions": { - "match_mapping_type": "string", - "match": "description", - "mapping": {"type": "text"}, - } - }, - { - "titles": { - "match_mapping_type": "string", - "match": "title", - "mapping": {"type": "text"}, - } - }, - # Projection Extension https://github.com/stac-extensions/projection - {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}}, - { - "proj_projjson": { - "match": "proj:projjson", - "mapping": {"type": "object", "enabled": False}, - } - }, - { - "proj_centroid": { - "match": "proj:centroid", - "mapping": {"type": "geo_point"}, - } - }, - { - "proj_geometry": { - "match": "proj:geometry", - "mapping": {"type": "object", "enabled": False}, - } - }, - { - "no_index_href": { - "match": "href", - "mapping": {"type": "text", "index": False}, - } - }, - # Default all other strings not otherwise specified to keyword - {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, -] - -ES_ITEMS_MAPPINGS = { - "numeric_detection": False, - "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, - "properties": { - "id": {"type": "keyword"}, - "collection": {"type": "keyword"}, - "geometry": {"type": "geo_shape"}, - "assets": {"type": "object", "enabled": False}, - "links": {"type": "object", "enabled": False}, - "properties": { - "type": "object", - "properties": { - # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md - "datetime": {"type": "date"}, - "start_datetime": {"type": "date"}, - "end_datetime": {"type": "date"}, - "created": {"type": "date"}, - "updated": {"type": "date"}, - # Satellite Extension https://github.com/stac-extensions/sat - "sat:absolute_orbit": {"type": "integer"}, - "sat:relative_orbit": {"type": "integer"}, - }, - }, - }, -} - -ES_COLLECTIONS_MAPPINGS = { - "numeric_detection": False, - "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, - "properties": { - "id": {"type": "keyword"}, - "extent.spatial.bbox": {"type": "long"}, - "extent.temporal.interval": {"type": "date"}, - "providers": {"type": "object", "enabled": False}, - "links": {"type": "object", "enabled": False}, - "item_assets": {"type": "object", "enabled": False}, - }, -} - - -def index_by_collection_id(collection_id: str) -> str: - """ - Translate a collection id into an Elasticsearch index name. - - Args: - collection_id (str): The collection id to translate into an index name. - - Returns: - str: The index name derived from the collection id. - """ - return f"{ITEMS_INDEX_PREFIX}{''.join(c for c in collection_id.lower() if c not in ES_INDEX_NAME_UNSUPPORTED_CHARS)}_{collection_id.encode('utf-8').hex()}" - - -def index_alias_by_collection_id(collection_id: str) -> str: - """ - Translate a collection id into an Elasticsearch index alias. - - Args: - collection_id (str): The collection id to translate into an index alias. - - Returns: - str: The index alias derived from the collection id. - """ - return f"{ITEMS_INDEX_PREFIX}{''.join(c for c in collection_id if c not in ES_INDEX_NAME_UNSUPPORTED_CHARS)}" - - -def indices(collection_ids: Optional[List[str]]) -> str: - """ - Get a comma-separated string of index names for a given list of collection ids. - - Args: - collection_ids: A list of collection ids. - - Returns: - A string of comma-separated index names. If `collection_ids` is None, returns the default indices. - """ - if collection_ids is None or collection_ids == []: - return ITEM_INDICES - else: - return ",".join([index_alias_by_collection_id(c) for c in collection_ids]) - async def create_index_templates() -> None: """ @@ -271,51 +123,6 @@ async def delete_item_index(collection_id: str): await client.close() -def mk_item_id(item_id: str, collection_id: str): - """Create the document id for an Item in Elasticsearch. - - Args: - item_id (str): The id of the Item. - collection_id (str): The id of the Collection that the Item belongs to. - - Returns: - str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character. - """ - return f"{item_id}|{collection_id}" - - -def mk_actions(collection_id: str, processed_items: List[Item]): - """Create Elasticsearch bulk actions for a list of processed items. - - Args: - collection_id (str): The identifier for the collection the items belong to. - processed_items (List[Item]): The list of processed items to be bulk indexed. - - Returns: - List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed, - each action being a dictionary with the following keys: - - `_index`: the index to store the document in. - - `_id`: the document's identifier. - - `_source`: the source of the document. - """ - return [ - { - "_index": index_alias_by_collection_id(collection_id), - "_id": mk_item_id(item["id"], item["collection"]), - "_source": item, - } - for item in processed_items - ] - - -# stac_pydantic classes extend _GeometryBase, which doesn't have a type field, -# So create our own Protocol for typing -# Union[ Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon, GeometryCollection] -class Geometry(Protocol): # noqa - type: str - coordinates: Any - - @attr.s class DatabaseLogic: """Database logic.""" @@ -466,7 +273,7 @@ async def get_one_item(self, collection_id: str, item_id: str) -> Dict: ) except exceptions.NotFoundError: raise NotFoundError( - f"Item {item_id} does not exist in Collection {collection_id}" + f"Item {item_id} does not exist inside Collection {collection_id}" ) return item["_source"] @@ -918,6 +725,24 @@ async def delete_item( f"Item {item_id} in collection {collection_id} not found" ) + async def get_items_mapping(self, collection_id: str) -> Dict[str, Any]: + """Get the mapping for the specified collection's items index. + + Args: + collection_id (str): The ID of the collection to get items mapping for. + + Returns: + Dict[str, Any]: The mapping information. + """ + index_name = index_alias_by_collection_id(collection_id) + try: + mapping = await self.client.indices.get_mapping( + index=index_name, allow_no_indices=False + ) + return mapping.body + except exceptions.NotFoundError: + raise NotFoundError(f"Mapping for index {index_name} not found") + async def create_collection(self, collection: Collection, refresh: bool = False): """Create a single collection in the database. @@ -1001,7 +826,7 @@ async def update_collection( "source": {"index": f"{ITEMS_INDEX_PREFIX}{collection_id}"}, "script": { "lang": "painless", - "source": f"""ctx._id = ctx._id.replace('{collection_id}', '{collection["id"]}'); ctx._source.collection = '{collection["id"]}' ;""", + "source": f"""ctx._id = ctx._id.replace('{collection_id}', '{collection["id"]}'); ctx._source.collection = '{collection["id"]}' ;""", # noqa: E702 }, }, wait_for_completion=True, diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 8be0eafd..90038302 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -39,13 +39,15 @@ settings = OpensearchSettings() session = Session.create_from_settings(settings) -filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient()) +database_logic = DatabaseLogic() + +filter_extension = FilterExtension( + client=EsAsyncBaseFiltersClient(database=database_logic) +) filter_extension.conformance_classes.append( "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" ) -database_logic = DatabaseLogic() - aggregation_extension = AggregationExtension( client=EsAsyncAggregationClient( database=database_logic, session=session, settings=settings diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 498c9c01..7bb7ac33 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -3,10 +3,9 @@ import asyncio import json import logging -import os from base64 import urlsafe_b64decode, urlsafe_b64encode from copy import deepcopy -from typing import Any, Dict, Iterable, List, Optional, Protocol, Tuple, Type, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Type import attr from opensearchpy import exceptions, helpers @@ -16,6 +15,21 @@ from starlette.requests import Request from stac_fastapi.core import serializers +from stac_fastapi.core.database_logic import ( + COLLECTIONS_INDEX, + DEFAULT_SORT, + ES_COLLECTIONS_MAPPINGS, + ES_ITEMS_MAPPINGS, + ES_ITEMS_SETTINGS, + ITEM_INDICES, + ITEMS_INDEX_PREFIX, + Geometry, + index_alias_by_collection_id, + index_by_collection_id, + indices, + mk_actions, + mk_item_id, +) from stac_fastapi.core.extensions import filter from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon from stac_fastapi.opensearch.config import ( @@ -27,168 +41,6 @@ logger = logging.getLogger(__name__) -NumType = Union[float, int] - -COLLECTIONS_INDEX = os.getenv("STAC_COLLECTIONS_INDEX", "collections") -ITEMS_INDEX_PREFIX = os.getenv("STAC_ITEMS_INDEX_PREFIX", "items_") -ES_INDEX_NAME_UNSUPPORTED_CHARS = { - "\\", - "/", - "*", - "?", - '"', - "<", - ">", - "|", - " ", - ",", - "#", - ":", -} - -ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}*" - -DEFAULT_SORT = { - "properties.datetime": {"order": "desc"}, - "id": {"order": "desc"}, - "collection": {"order": "desc"}, -} - -ES_ITEMS_SETTINGS = { - "index": { - "sort.field": list(DEFAULT_SORT.keys()), - "sort.order": [v["order"] for v in DEFAULT_SORT.values()], - } -} - -ES_MAPPINGS_DYNAMIC_TEMPLATES = [ - # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md - { - "descriptions": { - "match_mapping_type": "string", - "match": "description", - "mapping": {"type": "text"}, - } - }, - { - "titles": { - "match_mapping_type": "string", - "match": "title", - "mapping": {"type": "text"}, - } - }, - # Projection Extension https://github.com/stac-extensions/projection - {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}}, - { - "proj_projjson": { - "match": "proj:projjson", - "mapping": {"type": "object", "enabled": False}, - } - }, - { - "proj_centroid": { - "match": "proj:centroid", - "mapping": {"type": "geo_point"}, - } - }, - { - "proj_geometry": { - "match": "proj:geometry", - "mapping": {"type": "object", "enabled": False}, - } - }, - { - "no_index_href": { - "match": "href", - "mapping": {"type": "text", "index": False}, - } - }, - # Default all other strings not otherwise specified to keyword - {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, -] - -ES_ITEMS_MAPPINGS = { - "numeric_detection": False, - "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, - "properties": { - "id": {"type": "keyword"}, - "collection": {"type": "keyword"}, - "geometry": {"type": "geo_shape"}, - "assets": {"type": "object", "enabled": False}, - "links": {"type": "object", "enabled": False}, - "properties": { - "type": "object", - "properties": { - # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md - "datetime": {"type": "date"}, - "start_datetime": {"type": "date"}, - "end_datetime": {"type": "date"}, - "created": {"type": "date"}, - "updated": {"type": "date"}, - # Satellite Extension https://github.com/stac-extensions/sat - "sat:absolute_orbit": {"type": "integer"}, - "sat:relative_orbit": {"type": "integer"}, - }, - }, - }, -} - -ES_COLLECTIONS_MAPPINGS = { - "numeric_detection": False, - "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, - "properties": { - "id": {"type": "keyword"}, - "extent.spatial.bbox": {"type": "long"}, - "extent.temporal.interval": {"type": "date"}, - "providers": {"type": "object", "enabled": False}, - "links": {"type": "object", "enabled": False}, - "item_assets": {"type": "object", "enabled": False}, - }, -} - - -def index_by_collection_id(collection_id: str) -> str: - """ - Translate a collection id into an Elasticsearch index name. - - Args: - collection_id (str): The collection id to translate into an index name. - - Returns: - str: The index name derived from the collection id. - """ - return f"{ITEMS_INDEX_PREFIX}{''.join(c for c in collection_id.lower() if c not in ES_INDEX_NAME_UNSUPPORTED_CHARS)}_{collection_id.encode('utf-8').hex()}" - - -def index_alias_by_collection_id(collection_id: str) -> str: - """ - Translate a collection id into an Elasticsearch index alias. - - Args: - collection_id (str): The collection id to translate into an index alias. - - Returns: - str: The index alias derived from the collection id. - """ - return f"{ITEMS_INDEX_PREFIX}{''.join(c for c in collection_id if c not in ES_INDEX_NAME_UNSUPPORTED_CHARS)}" - - -def indices(collection_ids: Optional[List[str]]) -> str: - """ - Get a comma-separated string of index names for a given list of collection ids. - - Args: - collection_ids: A list of collection ids. - - Returns: - A string of comma-separated index names. If `collection_ids` is None, returns the default indices. - """ - if collection_ids is None or collection_ids == []: - return ITEM_INDICES - else: - return ",".join([index_alias_by_collection_id(c) for c in collection_ids]) - async def create_index_templates() -> None: """ @@ -292,51 +144,6 @@ async def delete_item_index(collection_id: str): await client.close() -def mk_item_id(item_id: str, collection_id: str): - """Create the document id for an Item in Elasticsearch. - - Args: - item_id (str): The id of the Item. - collection_id (str): The id of the Collection that the Item belongs to. - - Returns: - str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character. - """ - return f"{item_id}|{collection_id}" - - -def mk_actions(collection_id: str, processed_items: List[Item]): - """Create Elasticsearch bulk actions for a list of processed items. - - Args: - collection_id (str): The identifier for the collection the items belong to. - processed_items (List[Item]): The list of processed items to be bulk indexed. - - Returns: - List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed, - each action being a dictionary with the following keys: - - `_index`: the index to store the document in. - - `_id`: the document's identifier. - - `_source`: the source of the document. - """ - return [ - { - "_index": index_alias_by_collection_id(collection_id), - "_id": mk_item_id(item["id"], item["collection"]), - "_source": item, - } - for item in processed_items - ] - - -# stac_pydantic classes extend _GeometryBase, which doesn't have a type field, -# So create our own Protocol for typing -# Union[ Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon, GeometryCollection] -class Geometry(Protocol): # noqa - type: str - coordinates: Any - - @attr.s class DatabaseLogic: """Database logic.""" @@ -495,7 +302,7 @@ async def get_one_item(self, collection_id: str, item_id: str) -> Dict: ) except exceptions.NotFoundError: raise NotFoundError( - f"Item {item_id} does not exist in Collection {collection_id}" + f"Item {item_id} does not exist inside Collection {collection_id}" ) return item["_source"] @@ -950,6 +757,24 @@ async def delete_item( f"Item {item_id} in collection {collection_id} not found" ) + async def get_items_mapping(self, collection_id: str) -> Dict[str, Any]: + """Get the mapping for the specified collection's items index. + + Args: + collection_id (str): The ID of the collection to get items mapping for. + + Returns: + Dict[str, Any]: The mapping information. + """ + index_name = index_alias_by_collection_id(collection_id) + try: + mapping = await self.client.indices.get_mapping( + index=index_name, params={"allow_no_indices": "false"} + ) + return mapping + except exceptions.NotFoundError: + raise NotFoundError(f"Mapping for index {index_name} not found") + async def create_collection(self, collection: Collection, refresh: bool = False): """Create a single collection in the database. @@ -1033,7 +858,7 @@ async def update_collection( "source": {"index": f"{ITEMS_INDEX_PREFIX}{collection_id}"}, "script": { "lang": "painless", - "source": f"""ctx._id = ctx._id.replace('{collection_id}', '{collection["id"]}'); ctx._source.collection = '{collection["id"]}' ;""", + "source": f"""ctx._id = ctx._id.replace('{collection_id}', '{collection["id"]}'); ctx._source.collection = '{collection["id"]}' ;""", # noqa: E702 }, }, wait_for_completion=True, diff --git a/stac_fastapi/tests/rate_limit/test_rate_limit.py b/stac_fastapi/tests/rate_limit/test_rate_limit.py index fd6b5bce..4a7a7da5 100644 --- a/stac_fastapi/tests/rate_limit/test_rate_limit.py +++ b/stac_fastapi/tests/rate_limit/test_rate_limit.py @@ -18,7 +18,7 @@ async def test_rate_limit(app_client_rate_limit: AsyncClient, ctx): except RateLimitExceeded: status_code = 429 - logger.info(f"Request {i+1}: Status code {status_code}") + logger.info(f"Request {i + 1}: Status code {status_code}") assert ( status_code == expected_status_code ), f"Expected status code {expected_status_code}, but got {status_code}" @@ -32,7 +32,7 @@ async def test_rate_limit_no_limit(app_client: AsyncClient, ctx): response = await app_client.get("/collections") status_code = response.status_code - logger.info(f"Request {i+1}: Status code {status_code}") + logger.info(f"Request {i + 1}: Status code {status_code}") assert ( status_code == expected_status_code ), f"Expected status code {expected_status_code}, but got {status_code}" From fe376094e96576b3d3960a6c3faf19cd9f06073f Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Wed, 16 Apr 2025 23:09:29 +0800 Subject: [PATCH 2/8] Update stac-fastapi parent libraries to 5.1.1 (#354) **Related Issue(s):** - # **Description:** **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- .github/workflows/cicd.yml | 2 +- CHANGELOG.md | 8 ++- Makefile | 22 +++---- docker-compose.yml | 2 - stac_fastapi/core/setup.py | 8 +-- stac_fastapi/core/stac_fastapi/core/core.py | 57 +++++++++++-------- .../core/extensions/aggregation.py | 16 +++--- .../core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- .../stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- .../stac_fastapi/opensearch/version.py | 2 +- stac_fastapi/tests/resources/test_item.py | 5 +- 13 files changed, 71 insertions(+), 59 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index a966248b..864b52e3 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -65,7 +65,7 @@ jobs: strategy: matrix: - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13"] backend: [ "elasticsearch7", "elasticsearch8", "opensearch"] name: Python ${{ matrix.python-version }} testing with ${{ matrix.backend }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 04b4d793..f29766ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v4.0.0a0] + ### Added - Added support for dynamically-generated queryables based on Elasticsearch/OpenSearch mappings, with extensible metadata augmentation [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) - Included default queryables configuration for seamless integration. [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) @@ -14,6 +16,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Refactored database logic to reduce duplication [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) - Replaced `fastapi-slim` with `fastapi` dependency [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) +- Changed minimum Python version to 3.9 [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) +- Updated stac-fastapi api, types, and extensions libraries to 5.1.1 from 3.0.0 and made various associated changes [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) +- Changed makefile commands from 'docker-compose' to 'docker compose' [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) ### Fixed - Improved performance of `mk_actions` and `filter-links` methods [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) @@ -314,7 +319,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.5...main +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a0...main +[v4.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.5...v4.0.0a0 [v3.2.5]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.4...v3.2.5 [v3.2.4]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.3...v3.2.4 [v3.2.3]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.2...v3.2.3 diff --git a/Makefile b/Makefile index 9a3f23ce..e965d785 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ OS_APP_PORT ?= 8082 OS_HOST ?= docker.for.mac.localhost OS_PORT ?= 9202 -run_es = docker-compose \ +run_es = docker compose \ run \ -p ${EXTERNAL_APP_PORT}:${ES_APP_PORT} \ -e PY_IGNORE_IMPORTMISMATCH=1 \ @@ -18,7 +18,7 @@ run_es = docker-compose \ -e APP_PORT=${ES_APP_PORT} \ app-elasticsearch -run_os = docker-compose \ +run_os = docker compose \ run \ -p ${EXTERNAL_APP_PORT}:${OS_APP_PORT} \ -e PY_IGNORE_IMPORTMISMATCH=1 \ @@ -45,7 +45,7 @@ run-deploy-locally: .PHONY: image-dev image-dev: - docker-compose build + docker compose build .PHONY: docker-run-es docker-run-es: image-dev @@ -66,28 +66,28 @@ docker-shell-os: .PHONY: test-elasticsearch test-elasticsearch: -$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest' - docker-compose down + docker compose down .PHONY: test-opensearch test-opensearch: -$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest' - docker-compose down + docker compose down .PHONY: test test: -$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest' - docker-compose down + docker compose down -$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest' - docker-compose down + docker compose down .PHONY: run-database-es run-database-es: - docker-compose run --rm elasticsearch + docker compose run --rm elasticsearch .PHONY: run-database-os run-database-os: - docker-compose run --rm opensearch + docker compose run --rm opensearch .PHONY: pybase-install pybase-install: @@ -107,10 +107,10 @@ install-os: pybase-install .PHONY: docs-image docs-image: - docker-compose -f docker-compose.docs.yml \ + docker compose -f docker compose.docs.yml \ build .PHONY: docs docs: docs-image - docker-compose -f docker-compose.docs.yml \ + docker compose -f docker compose.docs.yml \ run docs \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index da4633b9..8ec0701b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.9' - services: app-elasticsearch: container_name: stac-fastapi-es diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index aedbe231..43b3e911 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -9,10 +9,10 @@ "fastapi", "attrs>=23.2.0", "pydantic", - "stac_pydantic>=3", - "stac-fastapi.types==3.0.0", - "stac-fastapi.api==3.0.0", - "stac-fastapi.extensions==3.0.0", + "stac_pydantic==3.1.*", + "stac-fastapi.api==5.1.1", + "stac-fastapi.extensions==5.1.1", + "stac-fastapi.types==5.1.1", "orjson", "overrides", "geojson-pydantic", diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 11bd34b4..1e96c371 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -37,7 +37,7 @@ from stac_fastapi.types.core import AsyncBaseCoreClient, AsyncBaseTransactionsClient from stac_fastapi.types.extension import ApiExtension from stac_fastapi.types.requests import get_base_url -from stac_fastapi.types.rfc3339 import DateTimeType +from stac_fastapi.types.rfc3339 import DateTimeType, rfc3339_str_to_datetime from stac_fastapi.types.search import BaseSearchPostRequest logger = logging.getLogger(__name__) @@ -277,7 +277,7 @@ async def item_collection( self, collection_id: str, bbox: Optional[BBox] = None, - datetime: Optional[DateTimeType] = None, + datetime: Optional[str] = None, limit: Optional[int] = 10, token: Optional[str] = None, **kwargs, @@ -287,7 +287,7 @@ async def item_collection( Args: collection_id (str): The identifier of the collection to read items from. bbox (Optional[BBox]): The bounding box to filter items by. - datetime (Optional[DateTimeType]): The datetime range to filter items by. + datetime (Optional[str]): The datetime range to filter items by. limit (int): The maximum number of items to return. The default value is 10. token (str): A token used for pagination. request (Request): The incoming request. @@ -426,23 +426,34 @@ def _return_date( return result - def _format_datetime_range(self, date_tuple: DateTimeType) -> str: + def _format_datetime_range(self, date_str: str) -> str: """ - Convert a tuple of datetime objects or None into a formatted string for API requests. + Convert a datetime range string into a normalized UTC string for API requests using rfc3339_str_to_datetime. Args: - date_tuple (tuple): A tuple containing two elements, each can be a datetime object or None. + date_str (str): A string containing two datetime values separated by a '/'. Returns: - str: A string formatted as 'YYYY-MM-DDTHH:MM:SS.sssZ/YYYY-MM-DDTHH:MM:SS.sssZ', with '..' used if any element is None. + str: A string formatted as 'YYYY-MM-DDTHH:MM:SSZ/YYYY-MM-DDTHH:MM:SSZ', with '..' used if any element is None. """ - def format_datetime(dt): - """Format a single datetime object to the ISO8601 extended format with 'Z'.""" - return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" if dt else ".." - - start, end = date_tuple - return f"{format_datetime(start)}/{format_datetime(end)}" + def normalize(dt): + dt = dt.strip() + if not dt or dt == "..": + return ".." + dt_obj = rfc3339_str_to_datetime(dt) + dt_utc = dt_obj.astimezone(timezone.utc) + return dt_utc.strftime("%Y-%m-%dT%H:%M:%SZ") + + if not isinstance(date_str, str): + return "../.." + if "/" not in date_str: + return f"{normalize(date_str)}/{normalize(date_str)}" + try: + start, end = date_str.split("/", 1) + except Exception: + return "../.." + return f"{normalize(start)}/{normalize(end)}" async def get_search( self, @@ -450,7 +461,7 @@ async def get_search( collections: Optional[List[str]] = None, ids: Optional[List[str]] = None, bbox: Optional[BBox] = None, - datetime: Optional[DateTimeType] = None, + datetime: Optional[str] = None, limit: Optional[int] = 10, query: Optional[str] = None, token: Optional[str] = None, @@ -458,7 +469,7 @@ async def get_search( sortby: Optional[str] = None, q: Optional[List[str]] = None, intersects: Optional[str] = None, - filter: Optional[str] = None, + filter_expr: Optional[str] = None, filter_lang: Optional[str] = None, **kwargs, ) -> stac_types.ItemCollection: @@ -468,7 +479,7 @@ async def get_search( collections (Optional[List[str]]): List of collection IDs to search in. ids (Optional[List[str]]): List of item IDs to search for. bbox (Optional[BBox]): Bounding box to search in. - datetime (Optional[DateTimeType]): Filter items based on the datetime field. + datetime (Optional[str]): Filter items based on the datetime field. limit (Optional[int]): Maximum number of results to return. query (Optional[str]): Query string to filter the results. token (Optional[str]): Access token to use when searching the catalog. @@ -495,7 +506,7 @@ async def get_search( } if datetime: - base_args["datetime"] = self._format_datetime_range(datetime) + base_args["datetime"] = self._format_datetime_range(date_str=datetime) if intersects: base_args["intersects"] = orjson.loads(unquote_plus(intersects)) @@ -506,12 +517,12 @@ async def get_search( for sort in sortby ] - if filter: - base_args["filter-lang"] = "cql2-json" + if filter_expr: + base_args["filter_lang"] = "cql2-json" base_args["filter"] = orjson.loads( - unquote_plus(filter) + unquote_plus(filter_expr) if filter_lang == "cql2-json" - else to_cql2(parse_cql2_text(filter)) + else to_cql2(parse_cql2_text(filter_expr)) ) if fields: @@ -593,8 +604,8 @@ async def post_search( ) # only cql2_json is supported here - if hasattr(search_request, "filter"): - cql2_filter = getattr(search_request, "filter", None) + if hasattr(search_request, "filter_expr"): + cql2_filter = getattr(search_request, "filter_expr", None) try: search = self.database.apply_cql2_filter(search, cql2_filter) except Exception as e: diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py index 2cf880c9..43bd543c 100644 --- a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py +++ b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py @@ -338,7 +338,7 @@ async def aggregate( datetime: Optional[DateTimeType] = None, intersects: Optional[str] = None, filter_lang: Optional[str] = None, - filter: Optional[str] = None, + filter_expr: Optional[str] = None, aggregations: Optional[str] = None, ids: Optional[List[str]] = None, bbox: Optional[BBox] = None, @@ -380,8 +380,8 @@ async def aggregate( if datetime: base_args["datetime"] = self._format_datetime_range(datetime) - if filter: - base_args["filter"] = self.get_filter(filter, filter_lang) + if filter_expr: + base_args["filter"] = self.get_filter(filter_expr, filter_lang) aggregate_request = EsAggregationExtensionPostRequest(**base_args) else: # Workaround for optional path param in POST requests @@ -389,9 +389,9 @@ async def aggregate( collection_id = path.split("/")[2] filter_lang = "cql2-json" - if aggregate_request.filter: - aggregate_request.filter = self.get_filter( - aggregate_request.filter, filter_lang + if aggregate_request.filter_expr: + aggregate_request.filter_expr = self.get_filter( + aggregate_request.filter_expr, filter_lang ) if collection_id: @@ -465,10 +465,10 @@ async def aggregate( detail=f"Aggregation {agg_name} not supported at catalog level", ) - if aggregate_request.filter: + if aggregate_request.filter_expr: try: search = self.database.apply_cql2_filter( - search, aggregate_request.filter + search, aggregate_request.filter_expr ) except Exception as e: raise HTTPException( diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index ca97d75a..3488c82b 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.2.5" +__version__ = "4.0.0a0" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 7fb82dc7..3355dbe3 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.2.5", + "stac-fastapi.core==4.0.0a0", "elasticsearch[async]==8.11.0", "elasticsearch-dsl==8.11.0", "uvicorn", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index ca97d75a..3488c82b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.2.5" +__version__ = "4.0.0a0" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 0befa10e..8cae5dce 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.2.5", + "stac-fastapi.core==4.0.0a0", "opensearch-py==2.4.2", "opensearch-py[async]==2.4.2", "uvicorn", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index ca97d75a..3488c82b 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.2.5" +__version__ = "4.0.0a0" diff --git a/stac_fastapi/tests/resources/test_item.py b/stac_fastapi/tests/resources/test_item.py index 904adbbf..5313b1fa 100644 --- a/stac_fastapi/tests/resources/test_item.py +++ b/stac_fastapi/tests/resources/test_item.py @@ -2,7 +2,7 @@ import os import uuid from copy import deepcopy -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from random import randint from urllib.parse import parse_qs, urlparse, urlsplit @@ -478,13 +478,10 @@ async def test_item_search_temporal_window_timezone_get( app_client, ctx, load_test_data ): """Test GET search with spatio-temporal query ending with Zulu and pagination(core)""" - tzinfo = timezone(timedelta(hours=1)) test_item = load_test_data("test_item.json") item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) item_date_before = item_date - timedelta(seconds=1) - item_date_before = item_date_before.replace(tzinfo=tzinfo) item_date_after = item_date + timedelta(seconds=1) - item_date_after = item_date_after.replace(tzinfo=tzinfo) params = { "collections": test_item["collection"], From e6ebe29b164d33b1ded1d3f74ebe16d4ff857007 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Thu, 17 Apr 2025 10:45:24 +0800 Subject: [PATCH 3/8] Inherit from Base Database Logic (#355) **Related Issue(s):** - #342 - #343 **Description:** **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- .github/workflows/deploy_mkdocs.yml | 4 ++-- CHANGELOG.md | 9 +++++++++ stac_fastapi/core/stac_fastapi/core/core.py | 17 ++++++----------- .../stac_fastapi/elasticsearch/config.py | 5 +++-- .../elasticsearch/database_logic.py | 3 ++- .../stac_fastapi/opensearch/config.py | 5 +++-- .../stac_fastapi/opensearch/database_logic.py | 3 ++- 7 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.github/workflows/deploy_mkdocs.yml b/.github/workflows/deploy_mkdocs.yml index 833c1021..3606d654 100644 --- a/.github/workflows/deploy_mkdocs.yml +++ b/.github/workflows/deploy_mkdocs.yml @@ -20,10 +20,10 @@ jobs: - name: Checkout main uses: actions/checkout@v4 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index f29766ff..4d555b72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +### Changed + +### Fixed + ## [v4.0.0a0] ### Added @@ -22,6 +28,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Improved performance of `mk_actions` and `filter-links` methods [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) +- Fixed inheritance relating to BaseDatabaseSettings and ApiBaseSettings [#355](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/355) +- Fixed delete_item and delete_collection methods return types [#355](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/355) +- Fixed inheritance relating to DatabaseLogic and BaseDatabaseLogic, and ApiBaseSettings [#355](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/355) ## [v3.2.5] - 2025-04-07 diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 1e96c371..16197da3 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -745,9 +745,7 @@ async def update_item( return ItemSerializer.db_to_stac(item, base_url) @overrides - async def delete_item( - self, item_id: str, collection_id: str, **kwargs - ) -> Optional[stac_types.Item]: + async def delete_item(self, item_id: str, collection_id: str, **kwargs) -> None: """Delete an item from a collection. Args: @@ -755,7 +753,7 @@ async def delete_item( collection_id (str): The identifier of the collection that contains the item. Returns: - Optional[stac_types.Item]: The deleted item, or `None` if the item was successfully deleted. + None: Returns 204 No Content on successful deletion """ await self.database.delete_item(item_id=item_id, collection_id=collection_id) return None @@ -825,23 +823,20 @@ async def update_collection( ) @overrides - async def delete_collection( - self, collection_id: str, **kwargs - ) -> Optional[stac_types.Collection]: + async def delete_collection(self, collection_id: str, **kwargs) -> None: """ Delete a collection. This method deletes an existing collection in the database. Args: - collection_id (str): The identifier of the collection that contains the item. - kwargs: Additional keyword arguments. + collection_id (str): The identifier of the collection to delete Returns: - None. + None: Returns 204 No Content on successful deletion Raises: - NotFoundError: If the collection doesn't exist. + NotFoundError: If the collection doesn't exist """ await self.database.delete_collection(collection_id=collection_id) return None diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index 0b1bcb5e..d14295f4 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -7,6 +7,7 @@ import certifi from elasticsearch import AsyncElasticsearch, Elasticsearch # type: ignore +from stac_fastapi.core.base_settings import ApiBaseSettings from stac_fastapi.types.config import ApiSettings @@ -69,7 +70,7 @@ def _es_config() -> Dict[str, Any]: _forbidden_fields: Set[str] = {"type"} -class ElasticsearchSettings(ApiSettings): +class ElasticsearchSettings(ApiSettings, ApiBaseSettings): """API settings.""" # Fields which are defined by STAC but not included in the database model @@ -82,7 +83,7 @@ def create_client(self): return Elasticsearch(**_es_config()) -class AsyncElasticsearchSettings(ApiSettings): +class AsyncElasticsearchSettings(ApiSettings, ApiBaseSettings): """API settings.""" # Fields which are defined by STAC but not included in the database model diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index c46b208d..38d05e29 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -12,6 +12,7 @@ from starlette.requests import Request from elasticsearch import exceptions, helpers # type: ignore +from stac_fastapi.core.base_database_logic import BaseDatabaseLogic from stac_fastapi.core.database_logic import ( COLLECTIONS_INDEX, DEFAULT_SORT, @@ -124,7 +125,7 @@ async def delete_item_index(collection_id: str): @attr.s -class DatabaseLogic: +class DatabaseLogic(BaseDatabaseLogic): """Database logic.""" client = AsyncElasticsearchSettings().create_client diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index 01551d94..6de2ab91 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -6,6 +6,7 @@ import certifi from opensearchpy import AsyncOpenSearch, OpenSearch +from stac_fastapi.core.base_settings import ApiBaseSettings from stac_fastapi.types.config import ApiSettings @@ -67,7 +68,7 @@ def _es_config() -> Dict[str, Any]: _forbidden_fields: Set[str] = {"type"} -class OpensearchSettings(ApiSettings): +class OpensearchSettings(ApiSettings, ApiBaseSettings): """API settings.""" # Fields which are defined by STAC but not included in the database model @@ -80,7 +81,7 @@ def create_client(self): return OpenSearch(**_es_config()) -class AsyncOpensearchSettings(ApiSettings): +class AsyncOpensearchSettings(ApiSettings, ApiBaseSettings): """API settings.""" # Fields which are defined by STAC but not included in the database model diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 7bb7ac33..22e6ffe0 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -15,6 +15,7 @@ from starlette.requests import Request from stac_fastapi.core import serializers +from stac_fastapi.core.base_database_logic import BaseDatabaseLogic from stac_fastapi.core.database_logic import ( COLLECTIONS_INDEX, DEFAULT_SORT, @@ -145,7 +146,7 @@ async def delete_item_index(collection_id: str): @attr.s -class DatabaseLogic: +class DatabaseLogic(BaseDatabaseLogic): """Database logic.""" client = AsyncSearchSettings().create_client From 790fb7d75c458d7fdd1cc828822df76fd32d603c Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Thu, 17 Apr 2025 10:46:43 +0800 Subject: [PATCH 4/8] Update CHANGELOG.md - add date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d555b72..ef0188bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed -## [v4.0.0a0] +## [v4.0.0a0] - 2025-04-17 ### Added - Added support for dynamically-generated queryables based on Elasticsearch/OpenSearch mappings, with extensible metadata augmentation [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) From 5f91e50c324829a043f988c711eb5a248f87a368 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Sat, 19 Apr 2025 01:02:37 +0800 Subject: [PATCH 5/8] Update package names for Pep625 compliance (#358) **Related Issue(s):** #337 **Description:** - Updated package names in setup.py files to use underscores instead of periods for PEP 625 compliance - Changed `stac_fastapi.opensearch` to `stac_fastapi_opensearch` - Changed `stac_fastapi.elasticsearch` to `stac_fastapi_elasticsearch` - Changed `stac_fastapi.core` to `stac_fastapi_core` - Updated all related dependencies to use the new naming convention - Renamed `docker-compose.yml` to `compose.yml` to align with Docker Compose V2 conventions - Removed deprecated `version` field from all compose files - Updated `STAC_FASTAPI_VERSION` environment variables to 4.0.0a1 in all compose files - Bumped version from 4.0.0a0 to 4.0.0a1 for the PEP 625 compliant release - Updated dependency requirements to use compatible release specifiers (~=) for more controlled updates while allowing for bug fixes and security patches - Removed elasticsearch-dsl dependency as it's now part of the elasticsearch package since version 8.18.0 **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 20 +++++++++-- Makefile | 4 +-- README.md | 32 ++++++++++-------- docker-compose.docs.yml => compose.docs.yml | 2 -- docker-compose.yml => compose.yml | 4 +-- ....basic_auth.yml => compose.basic_auth.yml} | 6 ++-- ...-compose.oauth2.yml => compose.oauth2.yml} | 6 ++-- ...ies.yml => compose.route_dependencies.yml} | 6 ++-- .../{docker-compose.yml => compose.yml} | 2 -- ....rate_limit.yml => compose.rate_limit.yml} | 6 ++-- stac_fastapi/core/setup.py | 23 +++++++------ .../core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 32 +++++++++--------- .../stac_fastapi/elasticsearch/config.py | 2 +- .../elasticsearch/database_logic.py | 6 ++-- .../stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 33 +++++++++---------- .../stac_fastapi/opensearch/version.py | 2 +- 18 files changed, 98 insertions(+), 92 deletions(-) rename docker-compose.docs.yml => compose.docs.yml (94%) rename docker-compose.yml => compose.yml (97%) rename examples/auth/{docker-compose.basic_auth.yml => compose.basic_auth.yml} (98%) rename examples/auth/{docker-compose.oauth2.yml => compose.oauth2.yml} (97%) rename examples/auth/{docker-compose.route_dependencies.yml => compose.route_dependencies.yml} (97%) rename examples/pip_docker/{docker-compose.yml => compose.yml} (98%) rename examples/rate_limit/{docker-compose.rate_limit.yml => compose.rate_limit.yml} (97%) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef0188bc..5b727634 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed -## [v4.0.0a0] - 2025-04-17 +## [v4.0.0a1] - 2925-04-17 + +### Changed +- Updated package names in setup.py files to use underscores instead of periods for PEP 625 compliance [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) + - Changed `stac_fastapi.opensearch` to `stac_fastapi_opensearch` + - Changed `stac_fastapi.elasticsearch` to `stac_fastapi_elasticsearch` + - Changed `stac_fastapi.core` to `stac_fastapi_core` + - Updated all related dependencies to use the new naming convention +- Renamed `docker-compose.yml` to `compose.yml` to align with Docker Compose V2 conventions [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) +- Removed deprecated `version` field from all compose files [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) +- Updated `STAC_FASTAPI_VERSION` environment variables to 4.0.0a1 in all compose files [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) +- Bumped version from 4.0.0a0 to 4.0.0a1 for the PEP 625 compliant release [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) +- Updated dependency requirements to use compatible release specifiers (~=) for more controlled updates while allowing for bug fixes and security patches [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/358) +- Removed elasticsearch-dsl dependency as it's now part of the elasticsearch package since version 8.18.0 [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/358) + +## [v4.0.0a0] - 2025-04-16 ### Added - Added support for dynamically-generated queryables based on Elasticsearch/OpenSearch mappings, with extensible metadata augmentation [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) @@ -328,7 +343,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a0...main +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a1...main +[v4.0.0a1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a0...v4.0.0a1 [v4.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.5...v4.0.0a0 [v3.2.5]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.4...v3.2.5 [v3.2.4]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.3...v3.2.4 diff --git a/Makefile b/Makefile index e965d785..a16fe6d9 100644 --- a/Makefile +++ b/Makefile @@ -107,10 +107,10 @@ install-os: pybase-install .PHONY: docs-image docs-image: - docker compose -f docker compose.docs.yml \ + docker compose -f compose.docs.yml \ build .PHONY: docs docs: docs-image - docker compose -f docker compose.docs.yml \ + docker compose -f compose.docs.yml \ run docs \ No newline at end of file diff --git a/README.md b/README.md index 84c38d12..d6e648f3 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

- [![PyPI version](https://badge.fury.io/py/stac-fastapi.elasticsearch.svg)](https://badge.fury.io/py/stac-fastapi.elasticsearch) + [![PyPI version](https://badge.fury.io/py/stac-fastapi-elasticsearch.svg)](https://badge.fury.io/py/stac-fastapi-elasticsearch) [![PyPI version](https://badge.fury.io/py/stac-fastapi-opensearch.svg)](https://badge.fury.io/py/stac-fastapi-opensearch) [![Join the chat at https://gitter.im/stac-fastapi-elasticsearch/community](https://badges.gitter.im/stac-fastapi-elasticsearch/community.svg)](https://gitter.im/stac-fastapi-elasticsearch/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -26,7 +26,7 @@ - Our Api core library can be used to create custom backends. See [stac-fastapi-mongo](https://github.com/Healy-Hyperspatial/stac-fastapi-mongo) for a working example. - Reach out on our [Gitter](https://app.gitter.im/#/room/#stac-fastapi-elasticsearch_community:gitter.im) channel or feel free to add to our [Discussions](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions) page here on github. -- There is [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) documentation here for examples on how to run some of the API routes locally - after starting the elasticsearch backend via the docker-compose.yml file. +- There is [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) documentation here for examples on how to run some of the API routes locally - after starting the elasticsearch backend via the compose.yml file. - The `/examples` folder shows an example of running stac-fastapi-elasticsearch from PyPI in docker without needing any code from the repository. There is also a Postman collection here that you can load into Postman for testing the API routes. - For changes, see the [Changelog](CHANGELOG.md) @@ -35,14 +35,20 @@ ### To install from PyPI: -```shell -pip install stac_fastapi.elasticsearch -``` -or -``` -pip install stac_fastapi.opensearch +```bash +# For versions 4.0.0a1 and newer (PEP 625 compliant naming): +pip install stac-fastapi-elasticsearch # Elasticsearch backend +pip install stac-fastapi-opensearch # Opensearch backend +pip install stac-fastapi-core # Core library + +# For versions 4.0.0a0 and older: +pip install stac-fastapi.elasticsearch # Elasticsearch backend +pip install stac-fastapi.opensearch # Opensearch backend +pip install stac-fastapi.core # Core library ``` +> **Important Note:** Starting with version 4.0.0a1, package names have changed from using periods (e.g., `stac-fastapi.core`) to using hyphens (e.g., `stac-fastapi-core`) to comply with PEP 625. The internal package structure uses underscores, but users should install with hyphens as shown above. Please update your requirements files accordingly. + ### To install and run via pre-built Docker Images We provide ready-to-use Docker images through GitHub Container Registry ([ElasticSearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pkgs/container/stac-fastapi-es) and [OpenSearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pkgs/container/stac-fastapi-os) backends). You can easily pull and run these images: @@ -57,15 +63,15 @@ docker pull ghcr.io/stac-utils/stac-fastapi-os:latest ## Run Elasticsearch API backend on localhost:8080 -You need to ensure [**Docker Compose**](https://docs.docker.com/compose/install/) or [**Podman Compose**](https://podman-desktop.io/docs/compose) installed and running on your machine. In the follwoing command instead of `docker-compose` you can use `podman-compose` as well. +You need to ensure [**Docker Compose**](https://docs.docker.com/compose/install/) or [**Podman Compose**](https://podman-desktop.io/docs/compose) installed and running on your machine. In the following command instead of `docker compose` you can use `podman-compose` as well. ```shell -docker-compose up elasticsearch app-elasticsearch +docker compose up elasticsearch app-elasticsearch ``` -By default, docker-compose uses Elasticsearch 8.x and OpenSearch 2.11.1. +By default, Docker Compose uses Elasticsearch 8.x and OpenSearch 2.11.1. If you wish to use a different version, put the following in a -file named `.env` in the same directory you run docker-compose from: +file named `.env` in the same directory you run Docker Compose from: ```shell ELASTICSEARCH_VERSION=7.17.1 @@ -165,7 +171,7 @@ These templates will be used implicitly when creating new Collection and Item in This section covers how to create a snapshot repository and then create and restore snapshots with this. Create a snapshot repository. This puts the files in the `elasticsearch/snapshots` in this git repo clone, as -the elasticsearch.yml and docker-compose files create a mapping from that directory to +the elasticsearch.yml and compose files create a mapping from that directory to `/usr/share/elasticsearch/snapshots` within the Elasticsearch container and grant permissions on using it. ```shell diff --git a/docker-compose.docs.yml b/compose.docs.yml similarity index 94% rename from docker-compose.docs.yml rename to compose.docs.yml index 4d91a06b..49573fbf 100644 --- a/docker-compose.docs.yml +++ b/compose.docs.yml @@ -1,5 +1,3 @@ -version: '3' - services: docs: container_name: stac-fastapi-docs-dev diff --git a/docker-compose.yml b/compose.yml similarity index 97% rename from docker-compose.yml rename to compose.yml index 8ec0701b..a66e584f 100644 --- a/docker-compose.yml +++ b/compose.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=2.1 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -41,7 +41,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=3.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/docker-compose.basic_auth.yml b/examples/auth/compose.basic_auth.yml similarity index 98% rename from examples/auth/docker-compose.basic_auth.yml rename to examples/auth/compose.basic_auth.yml index a6292a1f..c3e069ec 100644 --- a/examples/auth/docker-compose.basic_auth.yml +++ b/examples/auth/compose.basic_auth.yml @@ -1,5 +1,3 @@ -version: '3.9' - services: app-elasticsearch: container_name: stac-fastapi-es @@ -11,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=3.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -44,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=2.1 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/docker-compose.oauth2.yml b/examples/auth/compose.oauth2.yml similarity index 97% rename from examples/auth/docker-compose.oauth2.yml rename to examples/auth/compose.oauth2.yml index 8cd8f72f..ccd3bb1f 100644 --- a/examples/auth/docker-compose.oauth2.yml +++ b/examples/auth/compose.oauth2.yml @@ -1,5 +1,3 @@ -version: '3.9' - services: app-elasticsearch: container_name: stac-fastapi-es @@ -11,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=3.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -45,7 +43,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=2.1 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/docker-compose.route_dependencies.yml b/examples/auth/compose.route_dependencies.yml similarity index 97% rename from examples/auth/docker-compose.route_dependencies.yml rename to examples/auth/compose.route_dependencies.yml index b10fbb6f..0516fccd 100644 --- a/examples/auth/docker-compose.route_dependencies.yml +++ b/examples/auth/compose.route_dependencies.yml @@ -1,5 +1,3 @@ -version: '3.9' - services: app-elasticsearch: container_name: stac-fastapi-es @@ -11,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=3.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -44,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=2.1 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/pip_docker/docker-compose.yml b/examples/pip_docker/compose.yml similarity index 98% rename from examples/pip_docker/docker-compose.yml rename to examples/pip_docker/compose.yml index 3b2e6926..c9b3d641 100644 --- a/examples/pip_docker/docker-compose.yml +++ b/examples/pip_docker/compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: app-elasticsearch: container_name: stac-fastapi-es diff --git a/examples/rate_limit/docker-compose.rate_limit.yml b/examples/rate_limit/compose.rate_limit.yml similarity index 97% rename from examples/rate_limit/docker-compose.rate_limit.yml rename to examples/rate_limit/compose.rate_limit.yml index 5416e139..3fa902ab 100644 --- a/examples/rate_limit/docker-compose.rate_limit.yml +++ b/examples/rate_limit/compose.rate_limit.yml @@ -1,5 +1,3 @@ -version: '3.9' - services: app-elasticsearch: container_name: stac-fastapi-es @@ -11,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=2.1 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -44,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=3.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0a1 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 43b3e911..adde5c82 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -6,32 +6,31 @@ desc = f.read() install_requires = [ - "fastapi", + "fastapi~=0.109.0", "attrs>=23.2.0", - "pydantic", - "stac_pydantic==3.1.*", + "pydantic>=2.4.1,<3.0.0", + "stac_pydantic~=3.1.0", "stac-fastapi.api==5.1.1", "stac-fastapi.extensions==5.1.1", "stac-fastapi.types==5.1.1", - "orjson", - "overrides", - "geojson-pydantic", - "pygeofilter==0.3.1", - "jsonschema", - "slowapi==0.1.9", + "orjson~=3.9.0", + "overrides~=7.4.0", + "geojson-pydantic~=1.0.0", + "pygeofilter~=0.3.1", + "jsonschema~=4.0.0", + "slowapi~=0.1.9", ] setup( - name="stac_fastapi.core", + name="stac_fastapi_core", description="Core library for the Elasticsearch and Opensearch stac-fastapi backends.", long_description=desc, long_description_content_type="text/markdown", - python_requires=">=3.8", + python_requires=">=3.9", classifiers=[ "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index 3488c82b..af49b95b 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a0" +__version__ = "4.0.0a1" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 3355dbe3..1377211b 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,38 +6,36 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==4.0.0a0", - "elasticsearch[async]==8.11.0", - "elasticsearch-dsl==8.11.0", - "uvicorn", - "starlette", + "stac-fastapi-core==4.0.0a1", + "elasticsearch[async]~=8.18.0", + "uvicorn~=0.23.0", + "starlette>=0.35.0,<0.36.0", ] extra_reqs = { "dev": [ - "pytest", - "pytest-cov", - "pytest-asyncio", - "pre-commit", - "requests", - "ciso8601", - "httpx<=0.27.2", + "pytest~=7.0.0", + "pytest-cov~=4.0.0", + "pytest-asyncio~=0.21.0", + "pre-commit~=3.0.0", + "requests>=2.32.0,<3.0.0", + "ciso8601~=2.3.0", + "httpx>=0.24.0,<0.28.0", ], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], - "server": ["uvicorn[standard]==0.19.0"], + "docs": ["mkdocs~=1.4.0", "mkdocs-material~=9.0.0", "pdocs~=1.2.0"], + "server": ["uvicorn[standard]~=0.23.0"], } setup( - name="stac_fastapi.elasticsearch", + name="stac_fastapi_elasticsearch", description="An implementation of STAC API based on the FastAPI framework with both Elasticsearch and Opensearch.", long_description=desc, long_description_content_type="text/markdown", - python_requires=">=3.8", + python_requires=">=3.9", classifiers=[ "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index d14295f4..fb9e2e0f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -31,7 +31,7 @@ def _es_config() -> Dict[str, Any]: # Initialize the configuration dictionary config: Dict[str, Any] = { "hosts": hosts, - "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"}, + "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=8"}, } # Handle API key diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 38d05e29..ec84de57 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Type import attr -from elasticsearch_dsl import Q, Search +from elasticsearch.dsl import Q, Search from starlette.requests import Request from elasticsearch import exceptions, helpers # type: ignore @@ -232,7 +232,7 @@ async def get_all_collections( body={ "sort": [{"id": {"order": "asc"}}], "size": limit, - "search_after": search_after, + **({"search_after": search_after} if search_after is not None else {}), }, ) @@ -497,7 +497,7 @@ async def execute_search( ignore_unavailable=ignore_unavailable, query=query, sort=sort or DEFAULT_SORT, - search_after=search_after, + **({"search_after": search_after} if search_after is not None else {}), size=size_limit, ) ) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 3488c82b..af49b95b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a0" +__version__ = "4.0.0a1" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 8cae5dce..ece68679 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,38 +6,37 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==4.0.0a0", - "opensearch-py==2.4.2", - "opensearch-py[async]==2.4.2", - "uvicorn", - "starlette", + "stac-fastapi-core==4.0.0a1", + "opensearch-py~=2.8.0", + "opensearch-py[async]~=2.8.0", + "uvicorn~=0.23.0", + "starlette>=0.35.0,<0.36.0", ] extra_reqs = { "dev": [ - "pytest", - "pytest-cov", - "pytest-asyncio", - "pre-commit", - "requests", - "ciso8601", - "httpx<=0.27.2", + "pytest~=7.0.0", + "pytest-cov~=4.0.0", + "pytest-asyncio~=0.21.0", + "pre-commit~=3.0.0", + "requests>=2.32.0,<3.0.0", + "ciso8601~=2.3.0", + "httpx>=0.24.0,<0.28.0", ], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], - "server": ["uvicorn[standard]==0.19.0"], + "docs": ["mkdocs~=1.4.0", "mkdocs-material~=9.0.0", "pdocs~=1.2.0"], + "server": ["uvicorn[standard]~=0.23.0"], } setup( - name="stac_fastapi.opensearch", + name="stac_fastapi_opensearch", description="Opensearch stac-fastapi backend.", long_description=desc, long_description_content_type="text/markdown", - python_requires=">=3.8", + python_requires=">=3.9", classifiers=[ "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index 3488c82b..af49b95b 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a0" +__version__ = "4.0.0a1" From 67df17b4f36bef93715c30b45abd01f4791e6434 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Wed, 23 Apr 2025 00:01:51 +0800 Subject: [PATCH 6/8] Enable direct response, stac-fastapi 5.2.0, deprecation warnings (#359) **Related Issue(s):** - #347 **Description:** #### v4.0.0a2 release #### Added - Added support for high-performance direct response mode for both Elasticsearch and Opensearch backends, controlled by the `ENABLE_DIRECT_RESPONSE` environment variable. When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's jsonable_encoder and Pydantic serialization for significantly improved performance on large search responses. **Note:** In this mode, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes. Default is `false` for safety. A warning is logged at startup if enabled. See [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) #### Changed - Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). - Updated stac-fastapi parent libraries to 5.2.0. - Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. - Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. - Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. - Migrated startup event handling from deprecated `@app.on_event("startup")` to FastAPI's recommended lifespan context manager. This removes deprecation warnings and ensures compatibility with future FastAPI versions. **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 63 ++++++++++++------- README.md | 24 +++++-- compose.yml | 4 +- examples/auth/compose.basic_auth.yml | 4 +- examples/auth/compose.oauth2.yml | 4 +- examples/auth/compose.route_dependencies.yml | 4 +- examples/rate_limit/compose.rate_limit.yml | 4 +- stac_fastapi/core/setup.py | 6 +- stac_fastapi/core/stac_fastapi/core/core.py | 9 ++- .../stac_fastapi/core/extensions/query.py | 4 +- .../core/stac_fastapi/core/utilities.py | 29 +++++++++ .../core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- .../stac_fastapi/elasticsearch/app.py | 2 +- .../stac_fastapi/elasticsearch/config.py | 46 +++++++++++--- .../elasticsearch/database_logic.py | 28 ++++----- .../stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- .../opensearch/stac_fastapi/opensearch/app.py | 3 +- .../stac_fastapi/opensearch/config.py | 43 ++++++++++--- .../stac_fastapi/opensearch/database_logic.py | 46 ++++++-------- .../stac_fastapi/opensearch/version.py | 2 +- stac_fastapi/tests/api/test_api.py | 1 + stac_fastapi/tests/conftest.py | 21 ++++--- .../elasticsearch/test_direct_response.py | 39 ++++++++++++ 25 files changed, 277 insertions(+), 117 deletions(-) create mode 100644 stac_fastapi/tests/elasticsearch/test_direct_response.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b727634..06dd7791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed +### Fixed + +## [v4.0.0a2] - 2025-04-20 + +### Added +- Added support for high-performance direct response mode for both Elasticsearch and Opensearch backends, controlled by the `ENABLE_DIRECT_RESPONSE` environment variable. When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's jsonable_encoder and Pydantic serialization for significantly improved performance on large search responses. **Note:** In this mode, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes. Default is `false` for safety. A warning is logged at startup if enabled. See [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) and [PR #359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359). +- Added robust tests for the `ENABLE_DIRECT_RESPONSE` environment variable, covering both Elasticsearch and OpenSearch backends. Tests gracefully handle missing backends by attempting to import both configs and skipping if neither is available. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) + +### Changed +- Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Updated stac-fastapi parent libraries to 5.2.0. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated startup event handling from deprecated `@app.on_event("startup")` to FastAPI's recommended lifespan context manager. This removes deprecation warnings and ensures compatibility with future FastAPI versions. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Refactored all boolean environment variable parsing in both Elasticsearch and OpenSearch backends to use the shared `get_bool_env` utility. This ensures robust and consistent handling of environment variables such as `ES_USE_SSL`, `ES_HTTP_COMPRESS`, and `ES_VERIFY_CERTS` across both backends. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) + + ### Fixed ## [v4.0.0a1] - 2925-04-17 @@ -343,25 +361,26 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a1...main -[v4.0.0a1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a0...v4.0.0a1 -[v4.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.5...v4.0.0a0 -[v3.2.5]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.4...v3.2.5 -[v3.2.4]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.3...v3.2.4 -[v3.2.3]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.2...v3.2.3 -[v3.2.2]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.1...v3.2.2 -[v3.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.0...v3.2.1 -[v3.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.1.0...v3.2.0 -[v3.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.0.0...v3.1.0 -[v3.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.1...v3.0.0 -[v2.4.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.0...v2.4.1 -[v2.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.3.0...v2.4.0 -[v2.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.2.0...v2.3.0 -[v2.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.1.0...v2.2.0 -[v2.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.0.0...v2.1.0 -[v2.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.1.0...v2.0.0 -[v1.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.0.0...v1.1.0 -[v1.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.3.0...v1.0.0 -[v0.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.2.0...v0.3.0 -[v0.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0...v0.2.0 -[v0.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0 +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a2...main +[v4.0.0a2]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a1...v4.0.0a2 +[v4.0.0a1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a0...v4.0.0a1 +[v4.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.5...v4.0.0a0 +[v3.2.5]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.4...v3.2.5 +[v3.2.4]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.3...v3.2.4 +[v3.2.3]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.2...v3.2.3 +[v3.2.2]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.1...v3.2.2 +[v3.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.0...v3.2.1 +[v3.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.1.0...v3.2.0 +[v3.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.0.0...v3.1.0 +[v3.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.4.1...v3.0.0 +[v2.4.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.4.0...v2.4.1 +[v2.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.3.0...v2.4.0 +[v2.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.2.0...v2.3.0 +[v2.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.1.0...v2.2.0 +[v2.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.0.0...v2.1.0 +[v2.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v1.1.0...v2.0.0 +[v1.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v1.0.0...v1.1.0 +[v1.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.3.0...v1.0.0 +[v0.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.2.0...v0.3.0 +[v0.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.1.0...v0.2.0 +[v0.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.1.0 diff --git a/README.md b/README.md index d6e648f3..896db23f 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,18 @@ - There is [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) documentation here for examples on how to run some of the API routes locally - after starting the elasticsearch backend via the compose.yml file. - The `/examples` folder shows an example of running stac-fastapi-elasticsearch from PyPI in docker without needing any code from the repository. There is also a Postman collection here that you can load into Postman for testing the API routes. -- For changes, see the [Changelog](CHANGELOG.md) -- We are always welcoming contributions. For the development notes: [Contributing](CONTRIBUTING.md) + +### Performance Note + +The `enable_direct_response` option is provided by the stac-fastapi core library (introduced in stac-fastapi 5.2.0) and is available in this project starting from v4.0.0. + +**You can now control this setting via the `ENABLE_DIRECT_RESPONSE` environment variable.** + +When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's default serialization for improved performance. **However, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes.** + +This mode is best suited for public or read-only APIs where authentication and custom logic are not required. Default is `false` for safety. + +See: [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) ### To install from PyPI: @@ -74,8 +84,9 @@ If you wish to use a different version, put the following in a file named `.env` in the same directory you run Docker Compose from: ```shell -ELASTICSEARCH_VERSION=7.17.1 -OPENSEARCH_VERSION=2.11.0 +ELASTICSEARCH_VERSION=8.11.0 +OPENSEARCH_VERSION=2.11.1 +ENABLE_DIRECT_RESPONSE=false ``` The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information. @@ -100,8 +111,9 @@ You can customize additional settings in your `.env` file: | `RELOAD` | Enable auto-reload for development. | `true` | Optional | | `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional | | `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional | -| `ELASTICSEARCH_VERSION` | ElasticSearch version | `7.17.1` | Optional | -| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.0` | Optional | +| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional | +| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional | +| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional | > [!NOTE] > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, and `ES_VERIFY_CERTS` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch. diff --git a/compose.yml b/compose.yml index a66e584f..8f982ccb 100644 --- a/compose.yml +++ b/compose.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -41,7 +41,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.basic_auth.yml b/examples/auth/compose.basic_auth.yml index c3e069ec..88e95fa0 100644 --- a/examples/auth/compose.basic_auth.yml +++ b/examples/auth/compose.basic_auth.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.oauth2.yml b/examples/auth/compose.oauth2.yml index ccd3bb1f..3a295862 100644 --- a/examples/auth/compose.oauth2.yml +++ b/examples/auth/compose.oauth2.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -43,7 +43,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.route_dependencies.yml b/examples/auth/compose.route_dependencies.yml index 0516fccd..08576691 100644 --- a/examples/auth/compose.route_dependencies.yml +++ b/examples/auth/compose.route_dependencies.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/rate_limit/compose.rate_limit.yml b/examples/rate_limit/compose.rate_limit.yml index 3fa902ab..7d4340fb 100644 --- a/examples/rate_limit/compose.rate_limit.yml +++ b/examples/rate_limit/compose.rate_limit.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index adde5c82..ddf786b6 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -10,9 +10,9 @@ "attrs>=23.2.0", "pydantic>=2.4.1,<3.0.0", "stac_pydantic~=3.1.0", - "stac-fastapi.api==5.1.1", - "stac-fastapi.extensions==5.1.1", - "stac-fastapi.types==5.1.1", + "stac-fastapi.api==5.2.0", + "stac-fastapi.extensions==5.2.0", + "stac-fastapi.types==5.2.0", "orjson~=3.9.0", "overrides~=7.4.0", "geojson-pydantic~=1.0.0", diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 16197da3..3ac14efc 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -334,7 +334,7 @@ async def item_collection( search=search, limit=limit, sort=None, - token=token, # type: ignore + token=token, collection_ids=[collection_id], ) @@ -633,7 +633,7 @@ async def post_search( items, maybe_count, next_token = await self.database.execute_search( search=search, limit=limit, - token=search_request.token, # type: ignore + token=search_request.token, sort=sort, collection_ids=search_request.collections, ) @@ -701,7 +701,10 @@ async def create_item( database=self.database, settings=self.settings ) processed_items = [ - bulk_client.preprocess_item(item, base_url, BulkTransactionMethod.INSERT) for item in item["features"] # type: ignore + bulk_client.preprocess_item( + item, base_url, BulkTransactionMethod.INSERT + ) + for item in item["features"] ] await self.database.bulk_async( diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/query.py b/stac_fastapi/core/stac_fastapi/core/extensions/query.py index 3084cbf8..f6e0868d 100644 --- a/stac_fastapi/core/stac_fastapi/core/extensions/query.py +++ b/stac_fastapi/core/stac_fastapi/core/extensions/query.py @@ -10,7 +10,7 @@ from types import DynamicClassAttribute from typing import Any, Callable, Dict, Optional -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, model_validator from stac_pydantic.utils import AutoValueEnum from stac_fastapi.extensions.core.query import QueryExtension as QueryExtensionBase @@ -63,7 +63,7 @@ class QueryExtensionPostRequest(BaseModel): query: Optional[Dict[str, Dict[Operator, Any]]] = None - @root_validator(pre=True) + @model_validator(mode="before") def validate_query_fields(cls, values: Dict) -> Dict: """Validate query fields.""" ... diff --git a/stac_fastapi/core/stac_fastapi/core/utilities.py b/stac_fastapi/core/stac_fastapi/core/utilities.py index d8c69529..e7aafe67 100644 --- a/stac_fastapi/core/stac_fastapi/core/utilities.py +++ b/stac_fastapi/core/stac_fastapi/core/utilities.py @@ -3,6 +3,8 @@ This module contains functions for transforming geospatial coordinates, such as converting bounding boxes to polygon representations. """ +import logging +import os from typing import Any, Dict, List, Optional, Set, Union from stac_fastapi.types.stac import Item @@ -10,6 +12,33 @@ MAX_LIMIT = 10000 +def get_bool_env(name: str, default: bool = False) -> bool: + """ + Retrieve a boolean value from an environment variable. + + Args: + name (str): The name of the environment variable. + default (bool, optional): The default value to use if the variable is not set or unrecognized. Defaults to False. + + Returns: + bool: The boolean value parsed from the environment variable. + """ + value = os.getenv(name, str(default).lower()) + true_values = ("true", "1", "yes", "y") + false_values = ("false", "0", "no", "n") + if value.lower() in true_values: + return True + elif value.lower() in false_values: + return False + else: + logger = logging.getLogger(__name__) + logger.warning( + f"Environment variable '{name}' has unrecognized value '{value}'. " + f"Expected one of {true_values + false_values}. Using default: {default}" + ) + return default + + def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[float]]]: """Transform a bounding box represented by its four coordinates `b0`, `b1`, `b2`, and `b3` into a polygon. diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index af49b95b..2c71d558 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a1" +__version__ = "4.0.0a2" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 1377211b..77158e44 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==4.0.0a1", + "stac-fastapi-core==4.0.0a2", "elasticsearch[async]~=8.18.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 9510eaa6..91e239a4 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -87,7 +87,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "2.1"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0a2"), settings=settings, extensions=extensions, client=CoreClient( diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index fb9e2e0f..2044a4b2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -1,19 +1,22 @@ """API configuration.""" +import logging import os import ssl from typing import Any, Dict, Set import certifi +from elasticsearch._async.client import AsyncElasticsearch -from elasticsearch import AsyncElasticsearch, Elasticsearch # type: ignore +from elasticsearch import Elasticsearch # type: ignore[attr-defined] from stac_fastapi.core.base_settings import ApiBaseSettings +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.types.config import ApiSettings def _es_config() -> Dict[str, Any]: # Determine the scheme (http or https) - use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true" + use_ssl = get_bool_env("ES_USE_SSL", default=True) scheme = "https" if use_ssl else "http" # Configure the hosts parameter with the correct scheme @@ -44,7 +47,7 @@ def _es_config() -> Dict[str, Any]: config["headers"] = headers - http_compress = os.getenv("ES_HTTP_COMPRESS", "true").lower() == "true" + http_compress = get_bool_env("ES_HTTP_COMPRESS", default=True) if http_compress: config["http_compress"] = True @@ -53,8 +56,8 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.TLSVersion.TLSv1_3 # type: ignore - config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore + config["ssl_version"] = ssl.TLSVersion.TLSv1_3 + config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # Include CA Certificates if verifying certs if config["verify_certs"]: @@ -71,11 +74,18 @@ def _es_config() -> Dict[str, Any]: class ElasticsearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): @@ -84,13 +94,31 @@ def create_client(self): class AsyncElasticsearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): """Create async elasticsearch client.""" return AsyncElasticsearch(**_es_config()) + + +# Warn at import if direct response is enabled (applies to either settings class) +if ( + ElasticsearchSettings().enable_direct_response + or AsyncElasticsearchSettings().enable_direct_response +): + logging.basicConfig(level=logging.WARNING) + logging.warning( + "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" + ) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index ec84de57..f57ef9bb 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -8,10 +8,11 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Type import attr +import elasticsearch.helpers as helpers from elasticsearch.dsl import Q, Search +from elasticsearch.exceptions import NotFoundError as ESNotFoundError from starlette.requests import Request -from elasticsearch import exceptions, helpers # type: ignore from stac_fastapi.core.base_database_logic import BaseDatabaseLogic from stac_fastapi.core.database_logic import ( COLLECTIONS_INDEX, @@ -50,19 +51,18 @@ async def create_index_templates() -> None: """ client = AsyncElasticsearchSettings().create_client - await client.indices.put_template( + await client.indices.put_index_template( name=f"template_{COLLECTIONS_INDEX}", body={ "index_patterns": [f"{COLLECTIONS_INDEX}*"], - "mappings": ES_COLLECTIONS_MAPPINGS, + "template": {"mappings": ES_COLLECTIONS_MAPPINGS}, }, ) - await client.indices.put_template( + await client.indices.put_index_template( name=f"template_{ITEMS_INDEX_PREFIX}", body={ "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"], - "settings": ES_ITEMS_SETTINGS, - "mappings": ES_ITEMS_MAPPINGS, + "template": {"settings": ES_ITEMS_SETTINGS, "mappings": ES_ITEMS_MAPPINGS}, }, ) await client.close() @@ -80,7 +80,7 @@ async def create_collection_index() -> None: await client.options(ignore_status=400).indices.create( index=f"{COLLECTIONS_INDEX}-000001", - aliases={COLLECTIONS_INDEX: {}}, + body={"aliases": {COLLECTIONS_INDEX: {}}}, ) await client.close() @@ -100,7 +100,7 @@ async def create_item_index(collection_id: str): await client.options(ignore_status=400).indices.create( index=f"{index_by_collection_id(collection_id)}-000001", - aliases={index_alias_by_collection_id(collection_id): {}}, + body={"aliases": {index_alias_by_collection_id(collection_id): {}}}, ) await client.close() @@ -272,7 +272,7 @@ async def get_one_item(self, collection_id: str, item_id: str) -> Dict: index=index_alias_by_collection_id(collection_id), id=mk_item_id(item_id, collection_id), ) - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError( f"Item {item_id} does not exist inside Collection {collection_id}" ) @@ -512,7 +512,7 @@ async def execute_search( try: es_response = await search_task - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Collections '{collection_ids}' do not exist") hits = es_response["hits"]["hits"] @@ -595,7 +595,7 @@ def _fill_aggregation_parameters(name: str, agg: dict) -> dict: try: db_response = await search_task - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Collections '{collection_ids}' do not exist") return db_response @@ -721,7 +721,7 @@ async def delete_item( id=mk_item_id(item_id, collection_id), refresh=refresh, ) - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError( f"Item {item_id} in collection {collection_id} not found" ) @@ -741,7 +741,7 @@ async def get_items_mapping(self, collection_id: str) -> Dict[str, Any]: index=index_name, allow_no_indices=False ) return mapping.body - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Mapping for index {index_name} not found") async def create_collection(self, collection: Collection, refresh: bool = False): @@ -792,7 +792,7 @@ async def find_collection(self, collection_id: str) -> Collection: collection = await self.client.get( index=COLLECTIONS_INDEX, id=collection_id ) - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Collection {collection_id} not found") return collection["_source"] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index af49b95b..2c71d558 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a1" +__version__ = "4.0.0a2" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index ece68679..4d718733 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==4.0.0a1", + "stac-fastapi-core==4.0.0a2", "opensearch-py~=2.8.0", "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 90038302..504d5eab 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -87,7 +87,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "2.1"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0a2"), settings=settings, extensions=extensions, client=CoreClient( @@ -100,6 +100,7 @@ app = api.app app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") + # Add rate limit setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index 6de2ab91..00498468 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -1,4 +1,5 @@ """API configuration.""" +import logging import os import ssl from typing import Any, Dict, Set @@ -7,12 +8,13 @@ from opensearchpy import AsyncOpenSearch, OpenSearch from stac_fastapi.core.base_settings import ApiBaseSettings +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.types.config import ApiSettings def _es_config() -> Dict[str, Any]: # Determine the scheme (http or https) - use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true" + use_ssl = get_bool_env("ES_USE_SSL", default=True) scheme = "https" if use_ssl else "http" # Configure the hosts parameter with the correct scheme @@ -33,7 +35,7 @@ def _es_config() -> Dict[str, Any]: "headers": {"accept": "application/json", "Content-Type": "application/json"}, } - http_compress = os.getenv("ES_HTTP_COMPRESS", "true").lower() == "true" + http_compress = get_bool_env("ES_HTTP_COMPRESS", default=True) if http_compress: config["http_compress"] = True @@ -42,8 +44,8 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.PROTOCOL_SSLv23 # type: ignore - config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore + config["ssl_version"] = ssl.PROTOCOL_SSLv23 + config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # Include CA Certificates if verifying certs if config["verify_certs"]: @@ -69,11 +71,18 @@ def _es_config() -> Dict[str, Any]: class OpensearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): @@ -82,13 +91,31 @@ def create_client(self): class AsyncOpensearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): """Create async elasticsearch client.""" return AsyncOpenSearch(**_es_config()) + + +# Warn at import if direct response is enabled (applies to either settings class) +if ( + OpensearchSettings().enable_direct_response + or AsyncOpensearchSettings().enable_direct_response +): + logging.basicConfig(level=logging.WARNING) + logging.warning( + "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" + ) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 22e6ffe0..3184fa06 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -9,7 +9,6 @@ import attr from opensearchpy import exceptions, helpers -from opensearchpy.exceptions import TransportError from opensearchpy.helpers.query import Q from opensearchpy.helpers.search import Search from starlette.requests import Request @@ -80,24 +79,21 @@ async def create_collection_index() -> None: """ client = AsyncSearchSettings().create_client - search_body: Dict[str, Any] = { - "aliases": {COLLECTIONS_INDEX: {}}, - } - index = f"{COLLECTIONS_INDEX}-000001" - try: - await client.indices.create(index=index, body=search_body) - except TransportError as e: - if e.status_code == 400: - pass # Ignore 400 status codes - else: - raise e - + exists = await client.indices.exists(index=index) + if not exists: + await client.indices.create( + index=index, + body={ + "aliases": {COLLECTIONS_INDEX: {}}, + "mappings": ES_COLLECTIONS_MAPPINGS, + }, + ) await client.close() -async def create_item_index(collection_id: str): +async def create_item_index(collection_id: str) -> None: """ Create the index for Items. The settings of the index template will be used implicitly. @@ -109,24 +105,22 @@ async def create_item_index(collection_id: str): """ client = AsyncSearchSettings().create_client - search_body: Dict[str, Any] = { - "aliases": {index_alias_by_collection_id(collection_id): {}}, - } - try: + index_name = f"{index_by_collection_id(collection_id)}-000001" + exists = await client.indices.exists(index=index_name) + if not exists: await client.indices.create( - index=f"{index_by_collection_id(collection_id)}-000001", body=search_body + index=index_name, + body={ + "aliases": {index_alias_by_collection_id(collection_id): {}}, + "mappings": ES_ITEMS_MAPPINGS, + "settings": ES_ITEMS_SETTINGS, + }, ) - except TransportError as e: - if e.status_code == 400: - pass # Ignore 400 status codes - else: - raise e - await client.close() -async def delete_item_index(collection_id: str): +async def delete_item_index(collection_id: str) -> None: """Delete the index for items in a collection. Args: diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index af49b95b..2c71d558 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a1" +__version__ = "4.0.0a2" diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 64545807..fb128f74 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -7,6 +7,7 @@ ROUTES = { "GET /_mgmt/ping", + "GET /_mgmt/health", "GET /docs/oauth2-redirect", "HEAD /docs/oauth2-redirect", "GET /", diff --git a/stac_fastapi/tests/conftest.py b/stac_fastapi/tests/conftest.py index 651cdadb..a82f1485 100644 --- a/stac_fastapi/tests/conftest.py +++ b/stac_fastapi/tests/conftest.py @@ -8,7 +8,8 @@ import pytest import pytest_asyncio from fastapi import Depends, HTTPException, security, status -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient +from pydantic import ConfigDict from stac_pydantic import api from stac_fastapi.api.app import StacApi @@ -85,8 +86,7 @@ def __init__( class TestSettings(AsyncSettings): - class Config: - env_file = ".env.test" + model_config = ConfigDict(env_file=".env.test") settings = TestSettings() @@ -243,7 +243,9 @@ async def app_client(app): await create_index_templates() await create_collection_index() - async with AsyncClient(app=app, base_url="http://test-server") as c: + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test-server" + ) as c: yield c @@ -302,7 +304,9 @@ async def app_client_rate_limit(app_rate_limit): await create_index_templates() await create_collection_index() - async with AsyncClient(app=app_rate_limit, base_url="http://test-server") as c: + async with AsyncClient( + transport=ASGITransport(app=app_rate_limit), base_url="http://test-server" + ) as c: yield c @@ -392,7 +396,9 @@ async def app_client_basic_auth(app_basic_auth): await create_index_templates() await create_collection_index() - async with AsyncClient(app=app_basic_auth, base_url="http://test-server") as c: + async with AsyncClient( + transport=ASGITransport(app=app_basic_auth), base_url="http://test-server" + ) as c: yield c @@ -469,6 +475,7 @@ async def route_dependencies_client(route_dependencies_app): await create_collection_index() async with AsyncClient( - app=route_dependencies_app, base_url="http://test-server" + transport=ASGITransport(app=route_dependencies_app), + base_url="http://test-server", ) as c: yield c diff --git a/stac_fastapi/tests/elasticsearch/test_direct_response.py b/stac_fastapi/tests/elasticsearch/test_direct_response.py new file mode 100644 index 00000000..bbbceb56 --- /dev/null +++ b/stac_fastapi/tests/elasticsearch/test_direct_response.py @@ -0,0 +1,39 @@ +import importlib + +import pytest + + +def get_settings_class(): + """ + Try to import ElasticsearchSettings or OpenSearchSettings, whichever is available. + Returns a tuple: (settings_class, config_module) + """ + try: + config = importlib.import_module("stac_fastapi.elasticsearch.config") + importlib.reload(config) + return config.ElasticsearchSettings, config + except ModuleNotFoundError: + try: + config = importlib.import_module("stac_fastapi.opensearch.config") + importlib.reload(config) + return config.OpensearchSettings, config + except ModuleNotFoundError: + pytest.skip( + "Neither Elasticsearch nor OpenSearch config module is available." + ) + + +def test_enable_direct_response_true(monkeypatch): + """Test that ENABLE_DIRECT_RESPONSE env var enables direct response config.""" + monkeypatch.setenv("ENABLE_DIRECT_RESPONSE", "true") + settings_class, _ = get_settings_class() + settings = settings_class() + assert settings.enable_direct_response is True + + +def test_enable_direct_response_false(monkeypatch): + """Test that ENABLE_DIRECT_RESPONSE env var disables direct response config.""" + monkeypatch.setenv("ENABLE_DIRECT_RESPONSE", "false") + settings_class, _ = get_settings_class() + settings = settings_class() + assert settings.enable_direct_response is False From 807960723f92a536b3d3626272a017c63fd66607 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Wed, 23 Apr 2025 19:57:17 +0800 Subject: [PATCH 7/8] Lifespan context (#361) **Related Issue(s):** - # **Description:** - Migrated startup event handling from deprecated `@app.on_event("startup")` to FastAPI's recommended lifespan context manager. This removes deprecation warnings and ensures compatibility with future FastAPI versions **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- .../stac_fastapi/elasticsearch/app.py | 21 ++++++++++++------ .../opensearch/stac_fastapi/opensearch/app.py | 22 ++++++++++++------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 91e239a4..9aefca43 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -1,6 +1,9 @@ """FastAPI application.""" import os +from contextlib import asynccontextmanager + +from fastapi import FastAPI from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model @@ -97,17 +100,21 @@ search_post_request_model=post_request_model, route_dependencies=get_route_dependencies(), ) -app = api.app -app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") - -# Add rate limit -setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) -@app.on_event("startup") -async def _startup_event() -> None: +@asynccontextmanager +async def lifespan(app: FastAPI): + """Lifespan handler for FastAPI app. Initializes index templates and collections at startup.""" await create_index_templates() await create_collection_index() + yield + + +app = api.app +app.router.lifespan_context = lifespan +app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") +# Add rate limit +setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) def run() -> None: diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 504d5eab..07c48e67 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -1,6 +1,9 @@ """FastAPI application.""" import os +from contextlib import asynccontextmanager + +from fastapi import FastAPI from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model @@ -97,18 +100,21 @@ search_post_request_model=post_request_model, route_dependencies=get_route_dependencies(), ) -app = api.app -app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") - - -# Add rate limit -setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) -@app.on_event("startup") -async def _startup_event() -> None: +@asynccontextmanager +async def lifespan(app: FastAPI): + """Lifespan handler for FastAPI app. Initializes index templates and collections at startup.""" await create_index_templates() await create_collection_index() + yield + + +app = api.app +app.router.lifespan_context = lifespan +app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") +# Add rate limit +setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) def run() -> None: From 53a9d7ee891cb9522c345c4cd74892fb61a3c5cc Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Thu, 24 Apr 2025 10:47:12 +0800 Subject: [PATCH 8/8] Update to v4.0.0 (#362) **Description:** **Changes from 3.2.5:** #### Added - Added support for dynamically-generated queryables based on Elasticsearch/OpenSearch mappings, with extensible metadata augmentation [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) - Included default queryables configuration for seamless integration. [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) - Added support for high-performance direct response mode for both Elasticsearch and Opensearch backends, controlled by the `ENABLE_DIRECT_RESPONSE` environment variable. When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's jsonable_encoder and Pydantic serialization for significantly improved performance on large search responses. **Note:** In this mode, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes. Default is `false` for safety. A warning is logged at startup if enabled. See [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) and [PR #359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359). - Added robust tests for the `ENABLE_DIRECT_RESPONSE` environment variable, covering both Elasticsearch and OpenSearch backends. Tests gracefully handle missing backends by attempting to import both configs and skipping if neither is available. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) #### Changed - Refactored database logic to reduce duplication [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) - Replaced `fastapi-slim` with `fastapi` dependency [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) - Changed minimum Python version to 3.9 [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) - Updated stac-fastapi api, types, and extensions libraries to 5.1.1 from 3.0.0 and made various associated changes [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) - Changed makefile commands from 'docker-compose' to 'docker compose' [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) - Updated package names in setup.py files to use underscores instead of periods for PEP 625 compliance [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) - Changed `stac_fastapi.opensearch` to `stac_fastapi_opensearch` - Changed `stac_fastapi.elasticsearch` to `stac_fastapi_elasticsearch` - Changed `stac_fastapi.core` to `stac_fastapi_core` - Updated all related dependencies to use the new naming convention - Renamed `docker-compose.yml` to `compose.yml` to align with Docker Compose V2 conventions [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) - Removed deprecated `version` field from all compose files [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) - Updated `STAC_FASTAPI_VERSION` environment variables to 4.0.0 in all compose files [#362](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/362) - Bumped version from 4.0.0a2 to 4.0.0 for the PEP 625 compliant release [#362](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/362) - Updated dependency requirements to use compatible release specifiers (~=) for more controlled updates while allowing for bug fixes and security patches [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/358) - Removed elasticsearch-dsl dependency as it's now part of the elasticsearch package since version 8.18.0 [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/358) - Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Updated stac-fastapi parent libraries to 5.2.0. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Migrated startup event handling from deprecated `@app.on_event("startup")` to FastAPI's recommended lifespan context manager. This removes deprecation warnings and ensures compatibility with future FastAPI versions. [#361](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/361) - Refactored all boolean environment variable parsing in both Elasticsearch and OpenSearch backends to use the shared `get_bool_env` utility. This ensures robust and consistent handling of environment variables such as `ES_USE_SSL`, `ES_HTTP_COMPRESS`, and `ES_VERIFY_CERTS` across both backends. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) #### Fixed - Improved performance of `mk_actions` and `filter-links` methods [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) - Fixed inheritance relating to BaseDatabaseSettings and ApiBaseSettings [#355](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/355) - Fixed delete_item and delete_collection methods return types [#355](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/355) - Fixed inheritance relating to DatabaseLogic and BaseDatabaseLogic, and ApiBaseSettings [#355](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/355) **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 53 +++++++------------ compose.yml | 4 +- examples/auth/compose.basic_auth.yml | 4 +- examples/auth/compose.oauth2.yml | 4 +- examples/auth/compose.route_dependencies.yml | 4 +- examples/rate_limit/compose.rate_limit.yml | 4 +- .../core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- .../stac_fastapi/elasticsearch/app.py | 2 +- .../stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- .../opensearch/stac_fastapi/opensearch/app.py | 2 +- .../stac_fastapi/opensearch/version.py | 2 +- 13 files changed, 36 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06dd7791..1e68864e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,27 +13,20 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed -## [v4.0.0a2] - 2025-04-20 +## [v4.0.0] - 2025-04-23 ### Added +- Added support for dynamically-generated queryables based on Elasticsearch/OpenSearch mappings, with extensible metadata augmentation [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) +- Included default queryables configuration for seamless integration. [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) - Added support for high-performance direct response mode for both Elasticsearch and Opensearch backends, controlled by the `ENABLE_DIRECT_RESPONSE` environment variable. When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's jsonable_encoder and Pydantic serialization for significantly improved performance on large search responses. **Note:** In this mode, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes. Default is `false` for safety. A warning is logged at startup if enabled. See [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) and [PR #359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359). - Added robust tests for the `ENABLE_DIRECT_RESPONSE` environment variable, covering both Elasticsearch and OpenSearch backends. Tests gracefully handle missing backends by attempting to import both configs and skipping if neither is available. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) ### Changed -- Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) -- Updated stac-fastapi parent libraries to 5.2.0. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) -- Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) -- Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) -- Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) -- Migrated startup event handling from deprecated `@app.on_event("startup")` to FastAPI's recommended lifespan context manager. This removes deprecation warnings and ensures compatibility with future FastAPI versions. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) -- Refactored all boolean environment variable parsing in both Elasticsearch and OpenSearch backends to use the shared `get_bool_env` utility. This ensures robust and consistent handling of environment variables such as `ES_USE_SSL`, `ES_HTTP_COMPRESS`, and `ES_VERIFY_CERTS` across both backends. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - - -### Fixed - -## [v4.0.0a1] - 2925-04-17 - -### Changed +- Refactored database logic to reduce duplication [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) +- Replaced `fastapi-slim` with `fastapi` dependency [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) +- Changed minimum Python version to 3.9 [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) +- Updated stac-fastapi api, types, and extensions libraries to 5.1.1 from 3.0.0 and made various associated changes [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) +- Changed makefile commands from 'docker-compose' to 'docker compose' [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) - Updated package names in setup.py files to use underscores instead of periods for PEP 625 compliance [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) - Changed `stac_fastapi.opensearch` to `stac_fastapi_opensearch` - Changed `stac_fastapi.elasticsearch` to `stac_fastapi_elasticsearch` @@ -41,23 +34,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Updated all related dependencies to use the new naming convention - Renamed `docker-compose.yml` to `compose.yml` to align with Docker Compose V2 conventions [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) - Removed deprecated `version` field from all compose files [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) -- Updated `STAC_FASTAPI_VERSION` environment variables to 4.0.0a1 in all compose files [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) -- Bumped version from 4.0.0a0 to 4.0.0a1 for the PEP 625 compliant release [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/358) +- Updated `STAC_FASTAPI_VERSION` environment variables to 4.0.0 in all compose files [#362](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/362) +- Bumped version from 4.0.0a2 to 4.0.0 for the PEP 625 compliant release [#362](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/362) - Updated dependency requirements to use compatible release specifiers (~=) for more controlled updates while allowing for bug fixes and security patches [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/358) - Removed elasticsearch-dsl dependency as it's now part of the elasticsearch package since version 8.18.0 [#358](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/358) - -## [v4.0.0a0] - 2025-04-16 - -### Added -- Added support for dynamically-generated queryables based on Elasticsearch/OpenSearch mappings, with extensible metadata augmentation [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) -- Included default queryables configuration for seamless integration. [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) - -### Changed -- Refactored database logic to reduce duplication [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) -- Replaced `fastapi-slim` with `fastapi` dependency [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) -- Changed minimum Python version to 3.9 [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) -- Updated stac-fastapi api, types, and extensions libraries to 5.1.1 from 3.0.0 and made various associated changes [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) -- Changed makefile commands from 'docker-compose' to 'docker compose' [#354](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/354) +- Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Updated stac-fastapi parent libraries to 5.2.0. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated startup event handling from deprecated `@app.on_event("startup")` to FastAPI's recommended lifespan context manager. This removes deprecation warnings and ensures compatibility with future FastAPI versions. [#361](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/361) +- Refactored all boolean environment variable parsing in both Elasticsearch and OpenSearch backends to use the shared `get_bool_env` utility. This ensures robust and consistent handling of environment variables such as `ES_USE_SSL`, `ES_HTTP_COMPRESS`, and `ES_VERIFY_CERTS` across both backends. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) ### Fixed - Improved performance of `mk_actions` and `filter-links` methods [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351) @@ -361,10 +348,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a2...main -[v4.0.0a2]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a1...v4.0.0a2 -[v4.0.0a1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a0...v4.0.0a1 -[v4.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.5...v4.0.0a0 +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0...main +[v4.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.5...v4.0.0 [v3.2.5]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.4...v3.2.5 [v3.2.4]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.3...v3.2.4 [v3.2.3]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.2...v3.2.3 diff --git a/compose.yml b/compose.yml index 8f982ccb..24905483 100644 --- a/compose.yml +++ b/compose.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -41,7 +41,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.basic_auth.yml b/examples/auth/compose.basic_auth.yml index 88e95fa0..37de4013 100644 --- a/examples/auth/compose.basic_auth.yml +++ b/examples/auth/compose.basic_auth.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.oauth2.yml b/examples/auth/compose.oauth2.yml index 3a295862..09a3aa7b 100644 --- a/examples/auth/compose.oauth2.yml +++ b/examples/auth/compose.oauth2.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -43,7 +43,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.route_dependencies.yml b/examples/auth/compose.route_dependencies.yml index 08576691..da73e2bb 100644 --- a/examples/auth/compose.route_dependencies.yml +++ b/examples/auth/compose.route_dependencies.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/rate_limit/compose.rate_limit.yml b/examples/rate_limit/compose.rate_limit.yml index 7d4340fb..0f516dae 100644 --- a/examples/rate_limit/compose.rate_limit.yml +++ b/examples/rate_limit/compose.rate_limit.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a2 + - STAC_FASTAPI_VERSION=4.0.0 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index 2c71d558..6356730f 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a2" +__version__ = "4.0.0" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 77158e44..aa4a9371 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==4.0.0a2", + "stac-fastapi-core==4.0.0", "elasticsearch[async]~=8.18.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 9aefca43..9ccf009a 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -90,7 +90,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0a2"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0"), settings=settings, extensions=extensions, client=CoreClient( diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 2c71d558..6356730f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a2" +__version__ = "4.0.0" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 4d718733..c7427500 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==4.0.0a2", + "stac-fastapi-core==4.0.0", "opensearch-py~=2.8.0", "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 07c48e67..e7df7779 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -90,7 +90,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0a2"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0"), settings=settings, extensions=extensions, client=CoreClient( diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index 2c71d558..6356730f 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a2" +__version__ = "4.0.0" pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy