Skip to content

Commit a3d7d86

Browse files
stepankuzminrreusser
authored and
mourner
committed
Add support for caching partial responses (internal-1300)
* Add support for canonical `raster-array` URLs * Handle partial responses by keeping the range header in the query string * Rewrite query params manipulations using `URLSearchParams` * `RASTER_ARRAYS_URL_PREFIX` -> `RASTERARRAYS_URL_PREFIX` --------- Co-authored-by: Ricky Reusser <572717+rreusser@users.noreply.github.com>
1 parent 3b9ebec commit a3d7d86

File tree

7 files changed

+83
-44
lines changed

7 files changed

+83
-44
lines changed

‎debug/raster-array.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@
8080
map.on('style.load', () => {
8181
map.addSource('precipitations', {
8282
type: 'raster-array',
83-
url: [`https://api.mapbox.com/v4/mapboxsatellite.msm-precip-demo.json?access_token=${mapboxgl.accessToken}`],
83+
url: 'mapbox://mapboxsatellite.msm-precip-demo'
8484
});
8585

8686
map.on('sourcedata', (e) => {

‎src/source/raster_array_tile_source.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,9 @@ class RasterArrayTileSource extends RasterTileSource implements Source {
104104
delete tile.fbo;
105105
}
106106

107+
delete tile.request;
108+
delete tile.requestParams;
109+
107110
delete tile.neighboringTiles;
108111
tile.state = 'unloaded';
109112
}

‎src/util/config.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ type Config = {|
1414
REQUIRE_ACCESS_TOKEN: boolean,
1515
TILE_URL_VERSION: string,
1616
RASTER_URL_PREFIX: string,
17+
RASTERARRAYS_URL_PREFIX: string,
1718
ACCESS_TOKEN: ?string,
1819
MAX_PARALLEL_IMAGE_REQUESTS: number,
1920
DRACO_URL: string,
@@ -77,6 +78,7 @@ const config: Config = {
7778
FEEDBACK_URL: 'https://apps.mapbox.com/feedback',
7879
TILE_URL_VERSION: 'v4',
7980
RASTER_URL_PREFIX: 'raster/v1',
81+
RASTERARRAYS_URL_PREFIX: 'rasterarrays/v1',
8082
REQUIRE_ACCESS_TOKEN: true,
8183
ACCESS_TOKEN: null,
8284
DEFAULT_STYLE: 'mapbox://styles/mapbox/standard',

‎src/util/mapbox.js

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,8 @@ export class RequestManager {
142142

143143
if (urlObject.authority === 'raster') {
144144
urlObject.path = `/${config.RASTER_URL_PREFIX}${urlObject.path}`;
145+
} else if (urlObject.authority === 'rasterarrays') {
146+
urlObject.path = `/${config.RASTERARRAYS_URL_PREFIX}${urlObject.path}`;
145147
} else {
146148
const tileURLAPIPrefixRe = /^.+\/v4\//;
147149
urlObject.path = urlObject.path.replace(tileURLAPIPrefixRe, '/');
@@ -162,8 +164,8 @@ export class RequestManager {
162164

163165
const urlObject = parseUrl(url);
164166
// Make sure that we are dealing with a valid Mapbox tile URL.
165-
// Has to begin with /v4/ or /raster/v1, with a valid filename + extension
166-
if (!urlObject.path.match(/^(\/v4\/|\/raster\/v1\/)/) || !urlObject.path.match(extensionRe)) {
167+
// Has to begin with /v4/, /raster/v1 or /rasterarrays/v1 with a valid filename + extension
168+
if (!urlObject.path.match(/^(\/v4\/|\/(raster|rasterarrays)\/v1\/)/) || !urlObject.path.match(extensionRe)) {
167169
// Not a proper Mapbox tile URL.
168170
return url;
169171
}
@@ -173,6 +175,10 @@ export class RequestManager {
173175
// If the tile url has /raster/v1/, make the final URL mapbox://raster/....
174176
const rasterPrefix = `/${config.RASTER_URL_PREFIX}/`;
175177
result += `raster/${urlObject.path.replace(rasterPrefix, '')}`;
178+
} else if (urlObject.path.match(/^\/rasterarrays\/v1\//)) {
179+
// If the tile url has /rasterarrays/v1/, make the final URL mapbox://rasterarrays/....
180+
const rasterPrefix = `/${config.RASTERARRAYS_URL_PREFIX}/`;
181+
result += `rasterarrays/${urlObject.path.replace(rasterPrefix, '')}`;
176182
} else {
177183
const tilesPrefix = `/${config.TILE_URL_VERSION}/`;
178184
result += `tiles/${urlObject.path.replace(tilesPrefix, '')}`;

‎src/util/tile_request_cache.js

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -67,69 +67,90 @@ export function cachePut(request: Request, response: Response, requestTime: numb
6767
cacheOpen();
6868
if (!sharedCache) return;
6969

70+
const cacheControl = parseCacheControl(response.headers.get('Cache-Control') || '');
71+
if (cacheControl['no-store']) return;
72+
7073
const options: ResponseOptions = {
7174
status: response.status,
7275
statusText: response.statusText,
7376
headers: new Headers()
7477
};
78+
7579
response.headers.forEach((v, k) => options.headers.set(k, v));
7680

77-
const cacheControl = parseCacheControl(response.headers.get('Cache-Control') || '');
78-
if (cacheControl['no-store']) {
79-
return;
80-
}
8181
if (cacheControl['max-age']) {
8282
options.headers.set('Expires', new Date(requestTime + cacheControl['max-age'] * 1000).toUTCString());
8383
}
8484

8585
const expires = options.headers.get('Expires');
8686
if (!expires) return;
87+
8788
const timeUntilExpiry = new Date(expires).getTime() - requestTime;
8889
if (timeUntilExpiry < MIN_TIME_UNTIL_EXPIRY) return;
8990

91+
let strippedURL = stripQueryParameters(request.url);
92+
93+
// Handle partial responses by keeping the range header in the query string
94+
if (response.status === 206) {
95+
const range = request.headers.get('Range');
96+
if (!range) return;
97+
98+
options.status = 200;
99+
strippedURL = setQueryParameters(strippedURL, {range});
100+
}
101+
90102
prepareBody(response, body => {
91103
// $FlowFixMe[incompatible-call]
92104
const clonedResponse = new Response(body, options);
93105

94106
cacheOpen();
95107
if (!sharedCache) return;
96108
sharedCache
97-
.then(cache => cache.put(stripQueryParameters(request.url), clonedResponse))
109+
.then(cache => cache.put(strippedURL, clonedResponse))
98110
.catch(e => warnOnce(e.message));
99111
});
100112
}
101113

102-
function getQueryParameters(url: string) {
114+
function stripQueryParameters(url: string): string {
103115
const paramStart = url.indexOf('?');
104-
return paramStart > 0 ? url.slice(paramStart + 1).split('&') : [];
105-
}
106-
107-
function stripQueryParameters(url: string) {
108-
const start = url.indexOf('?');
109-
if (start < 0) return url;
116+
if (paramStart < 0) return url;
110117

111118
// preserve `language` and `worldview` params if any
112-
const params = getQueryParameters(url);
113-
const filteredParams = params.filter(param => {
114-
const entry = param.split('=');
115-
return entry[0] === 'language' || entry[0] === 'worldview';
116-
});
119+
const persistentParams = ['language', 'worldview'];
120+
121+
const nextParams = new URLSearchParams();
122+
const searchParams = new URLSearchParams(url.slice(paramStart));
123+
for (const param of persistentParams) {
124+
const value = searchParams.get(param);
125+
if (value) nextParams.set(param, value);
126+
}
127+
128+
return `${url.slice(0, paramStart)}?${nextParams.toString()}`;
129+
}
130+
131+
function setQueryParameters(url: string, params: {[string]: string}): string {
132+
const paramStart = url.indexOf('?');
133+
if (paramStart < 0) return `${url}?${new URLSearchParams(params).toString()}`;
117134

118-
if (filteredParams.length) {
119-
return `${url.slice(0, start)}?${filteredParams.join('&')}`;
135+
const searchParams = new URLSearchParams(url.slice(paramStart));
136+
for (const key in params) {
137+
searchParams.set(key, params[key]);
120138
}
121139

122-
return url.slice(0, start);
140+
return `${url.slice(0, paramStart)}?${searchParams.toString()}`;
123141
}
124142

125143
export function cacheGet(request: Request, callback: (error: ?any, response: ?Response, fresh: ?boolean) => void): void {
126144
cacheOpen();
127145
if (!sharedCache) return callback(null);
128146

129-
const strippedURL = stripQueryParameters(request.url);
130-
131-
((sharedCache: any): Promise<Cache>)
147+
sharedCache
132148
.then(cache => {
149+
let strippedURL = stripQueryParameters(request.url);
150+
151+
const range = request.headers.get('Range');
152+
if (range) strippedURL = setQueryParameters(strippedURL, {range});
153+
133154
// manually strip URL instead of `ignoreSearch: true` because of a known
134155
// performance issue in Chrome https://github.com/mapbox/mapbox-gl-js/issues/8431
135156
cache.match(strippedURL)
@@ -148,7 +169,6 @@ export function cacheGet(request: Request, callback: (error: ?any, response: ?Re
148169
.catch(callback);
149170
})
150171
.catch(callback);
151-
152172
}
153173

154174
function isFresh(response: Response) {

‎test/unit/util/mapbox.test.js

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,9 @@ describe("mapbox", () => {
365365
expect(
366366
manager.canonicalizeTileURL("http://api.mapbox.com/raster/v1/a.b/{z}/{x}/{y}.png?access_token=key", tileJSONURL)
367367
).toEqual("mapbox://raster/a.b/{z}/{x}/{y}.png");
368+
expect(
369+
manager.canonicalizeTileURL("http://api.mapbox.com/rasterarrays/v1/a.b/{z}/{x}/{y}.mrt?access_token=key", tileJSONURL)
370+
).toEqual("mapbox://rasterarrays/a.b/{z}/{x}/{y}.mrt");
368371

369372
// We don't ever expect to see these inputs, but be safe anyway.
370373
expect(manager.canonicalizeTileURL("http://path")).toEqual("http://path");
@@ -470,6 +473,9 @@ describe("mapbox", () => {
470473
expect(manager.normalizeTileURL("mapbox://raster/a.b/0/0/0.png")).toEqual(
471474
`https://api.mapbox.com/raster/v1/a.b/0/0/0.png?sku=${manager._skuToken}&access_token=key`
472475
);
476+
expect(manager.normalizeTileURL("mapbox://rasterarrays/a.b/0/0/0.mrt")).toEqual(
477+
`https://api.mapbox.com/rasterarrays/v1/a.b/0/0/0.mrt?sku=${manager._skuToken}&access_token=key`
478+
);
473479

474480
config.API_URL = 'https://api.example.com/';
475481
expect(manager.normalizeTileURL("mapbox://tiles/a.b/0/0/0.png")).toEqual(

‎test/unit/util/tile_request_cache.test.js

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import {describe, test, beforeEach, expect, vi} from "../../util/vitest.js";
1+
import {describe, test, beforeEach, expect, vi} from '../../util/vitest.js';
22
import {cacheGet, cachePut, cacheClose} from '../../../src/util/tile_request_cache.js';
33

44
describe('tile_request_cache', () => {
@@ -30,29 +30,31 @@ describe('tile_request_cache', () => {
3030
test('cacheGet, cache open error', () => {
3131
window.caches.open = vi.fn().mockRejectedValue(new Error('The operation is insecure'));
3232

33-
cacheGet({url:''}, (error) => {
33+
cacheGet(new Request(''), (error) => {
3434
expect(error).toBeTruthy();
3535
expect(error.message).toEqual('The operation is insecure');
3636
});
3737
});
3838

3939
test('cacheGet, cache match error', () => {
4040
const fakeCache = vi.fn();
41+
const fakeURL = new Request('someurl').url;
4142
fakeCache.match = vi.fn().mockImplementation((url) => {
42-
if (url === 'someurl') {
43+
if (url === fakeURL) {
4344
return Promise.reject(new Error('ohno'));
4445
}
4546
});
4647
window.caches.open = vi.fn().mockResolvedValue(fakeCache);
4748

48-
cacheGet({url:'someurl'}, (error) => {
49+
cacheGet(new Request(fakeURL), (error) => {
4950
expect(error).toBeTruthy();
5051
expect(error.message).toEqual('ohno');
5152
});
5253
});
5354

5455
test('cacheGet, happy path', async () => {
55-
const fakeResponse = {
56+
const cachedRequest = new Request(`someurl?language=es&worldview=US&range=${encodeURIComponent('bytes=0-')}`);
57+
const cachedResponse = {
5658
headers: {get: vi.fn().mockImplementation((name) => {
5759
switch (name) {
5860
case 'Expires':
@@ -61,33 +63,33 @@ describe('tile_request_cache', () => {
6163
return null;
6264
}
6365
})},
64-
clone: vi.fn().mockImplementation(() => fakeResponse),
66+
clone: vi.fn().mockImplementation(() => cachedResponse),
6567
body: 'yay'
6668
};
6769

68-
const fakeURL = 'someurl?language="es"&worldview="US"';
6970
const fakeCache = vi.fn();
70-
fakeCache.match = vi.fn().mockImplementation(async (url) => {
71-
if (url === fakeURL) {
72-
return fakeResponse;
73-
}
74-
});
71+
fakeCache.match = vi.fn().mockImplementation(async (url) =>
72+
url === cachedRequest.url ? cachedResponse : undefined
73+
);
7574
fakeCache.delete = vi.fn();
7675
fakeCache.put = vi.fn();
7776

7877
window.caches.open = vi.fn().mockImplementation(() => Promise.resolve(fakeCache));
7978

8079
await new Promise(resolve => {
80+
// ensure that the language and worldview query parameters are retained,
81+
// the Range header is added to the query string, but other query parameters are stripped
82+
const request = new Request(`someurl?language=es&worldview=US&accessToken=foo`);
83+
request.headers.set('Range', 'bytes=0-');
8184

82-
// ensure that the language and worldview query parameters are retained but other query parameters aren't
83-
cacheGet({url: `${fakeURL}&accessToken="foo"`}, (error, response, fresh) => {
85+
cacheGet(request, (error, response, fresh) => {
8486
expect(error).toBeFalsy();
85-
expect(fakeCache.match).toHaveBeenCalledWith(fakeURL);
86-
expect(fakeCache.delete).toHaveBeenCalledWith(fakeURL);
87+
expect(fakeCache.match).toHaveBeenCalledWith(cachedRequest.url);
88+
expect(fakeCache.delete).toHaveBeenCalledWith(cachedRequest.url);
8789
expect(response).toBeTruthy();
8890
expect(response.body).toEqual('yay');
8991
expect(fresh).toBeTruthy();
90-
expect(fakeCache.put).toHaveBeenCalledWith(fakeURL, fakeResponse);
92+
expect(fakeCache.put).toHaveBeenCalledWith(cachedRequest.url, cachedResponse);
9193
resolve();
9294
});
9395
});

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy