Skip to content

Commit 2ef0269

Browse files
ahuarte47ahuarte47
authored
Fix prev when reading first token_type=prev (#141)
* Fix prev when reading first token_type=prev * Version 0.6.7 (#141)
1 parent 5669749 commit 2ef0269

File tree

8 files changed

+3069
-4
lines changed

8 files changed

+3069
-4
lines changed

‎CHANGELOG.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](http://keepachangelog.com/)
66
and this project adheres to [Semantic Versioning](http://semver.org/).
77

8+
## [v0.6.7]
9+
10+
### Fixed
11+
- Fix not-null `prev` attribute when reading first token_type="prev" [#140](https://github.com/stac-utils/pgstac/issues/140)
12+
813
## [v0.6.6]
914

1015
### Added
@@ -206,7 +211,8 @@ _TODO_
206211

207212
- Fixed issue with pypgstac loads which caused some writes to fail ([#18](https://github.com/stac-utils/pgstac/pull/18))
208213

209-
[unreleased]: https://github.com/stac-utils/pgstac/compare/v0.6.6...HEAD
214+
[unreleased]: https://github.com/stac-utils/pgstac/compare/v0.6.7...HEAD
215+
[v0.6.7]: https://github.com//stac-utils/pgstac/compare/v0.6.6...v0.6.7
210216
[v0.6.6]: https://github.com//stac-utils/pgstac/compare/v0.6.5...v0.6.6
211217
[v0.6.5]: https://github.com//stac-utils/pgstac/compare/v0.6.4...v0.6.5
212218
[v0.6.4]: https://github.com//stac-utils/pgstac/compare/v0.6.3...v0.6.4

‎pypgstac/pypgstac/migrations/pgstac.0.6.6-0.6.7.sql‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
SET SEARCH_PATH to pgstac, public;
2+
set check_function_bodies = off;
3+
4+
CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$
5+
DECLARE
6+
searches searches%ROWTYPE;
7+
_where text;
8+
token_where text;
9+
full_where text;
10+
orderby text;
11+
query text;
12+
token_type text := substr(_search->>'token',1,4);
13+
_limit int := coalesce((_search->>'limit')::int, 10);
14+
curs refcursor;
15+
cntr int := 0;
16+
iter_record items%ROWTYPE;
17+
first_record jsonb;
18+
first_item items%ROWTYPE;
19+
last_item items%ROWTYPE;
20+
last_record jsonb;
21+
out_records jsonb := '[]'::jsonb;
22+
prev_query text;
23+
next text;
24+
prev_id text;
25+
has_next boolean := false;
26+
has_prev boolean := false;
27+
prev text;
28+
total_count bigint;
29+
context jsonb;
30+
collection jsonb;
31+
includes text[];
32+
excludes text[];
33+
exit_flag boolean := FALSE;
34+
batches int := 0;
35+
timer timestamptz := clock_timestamp();
36+
pstart timestamptz;
37+
pend timestamptz;
38+
pcurs refcursor;
39+
search_where search_wheres%ROWTYPE;
40+
id text;
41+
BEGIN
42+
CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP;
43+
-- if ids is set, short circuit and just use direct ids query for each id
44+
-- skip any paging or caching
45+
-- hard codes ordering in the same order as the array of ids
46+
IF _search ? 'ids' THEN
47+
INSERT INTO results
48+
SELECT
49+
CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN
50+
content_nonhydrated(items, _search->'fields')
51+
ELSE
52+
content_hydrate(items, _search->'fields')
53+
END
54+
FROM items WHERE
55+
items.id = ANY(to_text_array(_search->'ids'))
56+
AND
57+
CASE WHEN _search ? 'collections' THEN
58+
items.collection = ANY(to_text_array(_search->'collections'))
59+
ELSE TRUE
60+
END
61+
ORDER BY items.datetime desc, items.id desc
62+
;
63+
SELECT INTO total_count count(*) FROM results;
64+
ELSE
65+
searches := search_query(_search);
66+
_where := searches._where;
67+
orderby := searches.orderby;
68+
search_where := where_stats(_where);
69+
total_count := coalesce(search_where.total_count, search_where.estimated_count);
70+
71+
IF token_type='prev' THEN
72+
token_where := get_token_filter(_search, null::jsonb);
73+
orderby := sort_sqlorderby(_search, TRUE);
74+
END IF;
75+
IF token_type='next' THEN
76+
token_where := get_token_filter(_search, null::jsonb);
77+
END IF;
78+
79+
full_where := concat_ws(' AND ', _where, token_where);
80+
RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer;
81+
timer := clock_timestamp();
82+
83+
FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP
84+
timer := clock_timestamp();
85+
query := format('%s LIMIT %s', query, _limit + 1);
86+
RAISE NOTICE 'Partition Query: %', query;
87+
batches := batches + 1;
88+
-- curs = create_cursor(query);
89+
OPEN curs FOR EXECUTE query;
90+
LOOP
91+
FETCH curs into iter_record;
92+
EXIT WHEN NOT FOUND;
93+
cntr := cntr + 1;
94+
95+
IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN
96+
last_record := content_nonhydrated(iter_record, _search->'fields');
97+
ELSE
98+
last_record := content_hydrate(iter_record, _search->'fields');
99+
END IF;
100+
last_item := iter_record;
101+
IF cntr = 1 THEN
102+
first_item := last_item;
103+
first_record := last_record;
104+
END IF;
105+
IF cntr <= _limit THEN
106+
INSERT INTO results (content) VALUES (last_record);
107+
ELSIF cntr > _limit THEN
108+
has_next := true;
109+
exit_flag := true;
110+
EXIT;
111+
END IF;
112+
END LOOP;
113+
CLOSE curs;
114+
RAISE NOTICE 'Query took %.', clock_timestamp()-timer;
115+
timer := clock_timestamp();
116+
EXIT WHEN exit_flag;
117+
END LOOP;
118+
RAISE NOTICE 'Scanned through % partitions.', batches;
119+
END IF;
120+
121+
SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL;
122+
123+
DROP TABLE results;
124+
125+
126+
-- Flip things around if this was the result of a prev token query
127+
IF token_type='prev' THEN
128+
out_records := flip_jsonb_array(out_records);
129+
first_item := last_item;
130+
first_record := last_record;
131+
END IF;
132+
133+
-- If this query has a token, see if there is data before the first record
134+
IF _search ? 'token' THEN
135+
prev_query := format(
136+
'SELECT 1 FROM items WHERE %s LIMIT 1',
137+
concat_ws(
138+
' AND ',
139+
_where,
140+
trim(get_token_filter(_search, to_jsonb(first_item)))
141+
)
142+
);
143+
RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record;
144+
EXECUTE prev_query INTO has_prev;
145+
IF FOUND and has_prev IS NOT NULL THEN
146+
RAISE NOTICE 'Query results from prev query: %', has_prev;
147+
has_prev := TRUE;
148+
END IF;
149+
END IF;
150+
has_prev := COALESCE(has_prev, FALSE);
151+
152+
IF has_prev THEN
153+
prev := out_records->0->>'id';
154+
END IF;
155+
IF has_next OR token_type='prev' THEN
156+
next := out_records->-1->>'id';
157+
END IF;
158+
159+
IF context(_search->'conf') != 'off' THEN
160+
context := jsonb_strip_nulls(jsonb_build_object(
161+
'limit', _limit,
162+
'matched', total_count,
163+
'returned', coalesce(jsonb_array_length(out_records), 0)
164+
));
165+
ELSE
166+
context := jsonb_strip_nulls(jsonb_build_object(
167+
'limit', _limit,
168+
'returned', coalesce(jsonb_array_length(out_records), 0)
169+
));
170+
END IF;
171+
172+
collection := jsonb_build_object(
173+
'type', 'FeatureCollection',
174+
'features', coalesce(out_records, '[]'::jsonb),
175+
'next', next,
176+
'prev', prev,
177+
'context', context
178+
);
179+
180+
RETURN collection;
181+
END;
182+
$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public;
183+
184+
185+
SELECT set_version('0.6.7');

0 commit comments

Comments
 (0)