Skip to content

basemapper.py

Thread to handle downloads for Queue.

Parameters:

Name Type Description Default
dest str

The filespec of the tile cache

required
mirrors list

The list of mirrors to get imagery

required
tiles list

The list of tiles to download

required
xy bool

Whether to swap the X & Y fields in the TMS URL

required
Source code in osm_fieldwork/basemapper.py
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
def dlthread(
    dest: str,
    mirrors: list,
    tiles: list,
    xy: bool,
):
    """Thread to handle downloads for Queue.

    Args:
        dest (str): The filespec of the tile cache
        mirrors (list): The list of mirrors to get imagery
        tiles (list): The list of tiles to download
        xy (bool): Whether to swap the X & Y fields in the TMS URL
    """
    if len(tiles) == 0:
        # epdb.st()
        return
    # counter = -1

    # start = datetime.now()

    # totaltime = 0.0
    log.info("Downloading %d tiles in thread %d to %s" % (len(tiles), threading.get_ident(), dest))
    for tile in tiles:
        filespec = f"{tile[2]}/{tile[1]}/{tile[0]}"
        for site in mirrors:
            if site["source"] != "topo":
                filespec += "." + site["suffix"]
            url = site["url"]
            if site["source"] == "bing":
                bingkey = mercantile.quadkey(tile)
                remote = url % bingkey
            elif site["source"] == "google":
                path = f"x={tile[0]}&s=&y={tile[1]}&z={tile[2]}"
                remote = url % path
            elif site["source"] == "custom":
                if not xy:
                    # z/y/x format
                    path = f"{tile[2]}/{tile[1]}/{tile[0]}"
                else:
                    # z/x/y format
                    path = f"{tile[2]}/{tile[0]}/{tile[1]}"
                remote = url % path
            else:
                remote = url % filespec
            print("Getting file from: %s" % remote)
            # Create the subdirectories as pySmartDL doesn't do it for us
            Path(dest).mkdir(parents=True, exist_ok=True)

        dl = None
        try:
            if site["source"] == "topo":
                filespec += "." + site["suffix"]
            outfile = dest + "/" + filespec
            if not Path(outfile).exists():
                dl = SmartDL(remote, dest=outfile, connect_default_logger=False)
                dl.start()
            else:
                log.debug("%s exists!" % (outfile))
        except Exception as e:
            log.error(e)
            if dl:
                log.error(f"Couldn't download {filespec}: {dl.get_errors()}")
            else:
                log.error(f"Couldn't download {filespec}")

options: show_source: false heading_level: 3

Bases: object

Basemapper parent class.

Parameters:

Name Type Description Default
boundary Union[str, BytesIO]

A BBOX string or GeoJSON provided as BytesIO object of the AOI. The GeoJSON can contain multiple geometries.

required
base str

The base directory to cache map tile in

required
source str

The upstream data source for map tiles

required
xy bool

Whether to swap the X & Y fields in the TMS URL

required

Returns:

Type Description
BaseMapper

An instance of this class

Source code in osm_fieldwork/basemapper.py
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
def __init__(
    self,
    boundary: Union[str, BytesIO],
    base: str,
    source: str,
    xy: bool,
):
    """Create an tile basemap for ODK Collect.

    Args:
        boundary (Union[str, BytesIO]): A BBOX string or GeoJSON provided as BytesIO object of the AOI.
            The GeoJSON can contain multiple geometries.
        base (str): The base directory to cache map tile in
        source (str): The upstream data source for map tiles
        xy (bool): Whether to swap the X & Y fields in the TMS URL

    Returns:
        (BaseMapper): An instance of this class
    """
    bbox_factory = BoundaryHandlerFactory(boundary)
    self.bbox = bbox_factory.get_bounding_box()
    self.tiles = list()
    self.base = base
    # sources for imagery
    self.source = source
    self.sources = dict()
    self.xy = xy

    path = xlsforms_path.replace("xlsforms", "imagery.yaml")
    self.yaml = YamlFile(path)

    for entry in self.yaml.yaml["sources"]:
        for k, v in entry.items():
            src = dict()
            for item in v:
                src["source"] = k
                for k1, v1 in item.items():
                    # print(f"\tFIXME2: {k1} - {v1}")
                    src[k1] = v1
            self.sources[k] = src

customTMS

customTMS(url, name='custom', source='custom', suffix='jpg')

Add a custom TMS URL to the list of sources.

The url must end in %s to be replaced with the tile xyz values.

Format examples: https://basemap.nationalmap.gov/ArcGIS/rest/services/USGSTopo/MapServer/tile/{z}/{y}/{x} https://maps.nyc.gov/xyz/1.0.0/carto/basemap/%s https://maps.nyc.gov/xyz/1.0.0/carto/basemap/{z}/{x}/{y}.jpg

The method will replace {z}/{x}/{y}.jpg with %s

Parameters:

Name Type Description Default
name str

The name to display

'custom'
url str

The URL string

required
suffix str

The suffix, png or jpg

'jpg'
source str

The source value to use as an index

'custom'
Source code in osm_fieldwork/basemapper.py
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
def customTMS(self, url: str, name: str = "custom", source: str = "custom", suffix: str = "jpg"):
    """Add a custom TMS URL to the list of sources.

    The url must end in %s to be replaced with the tile xyz values.

    Format examples:
    https://basemap.nationalmap.gov/ArcGIS/rest/services/USGSTopo/MapServer/tile/{z}/{y}/{x}
    https://maps.nyc.gov/xyz/1.0.0/carto/basemap/%s
    https://maps.nyc.gov/xyz/1.0.0/carto/basemap/{z}/{x}/{y}.jpg

    The method will replace {z}/{x}/{y}.jpg with %s

    Args:
        name (str): The name to display
        url (str): The URL string
        suffix (str): The suffix, png or jpg
        source (str): The source value to use as an index
    """
    # Remove any file extensions if present and update the 'suffix' parameter
    if url.endswith(".jpg"):
        source = "jpg"
        suffix = "jpg"
        url = url[:-4]  # Remove the last 4 characters (".jpg")
    elif url.endswith(".png"):
        source = "png"
        suffix = "png"
        url = url[:-4]  # Remove the last 4 characters (".png")

    # Replace "{z}/{x}/{y}" with "%s"
    url = re.sub(r"/{[xyz]+\}", "", url)
    url = url + r"/%s"

    tms_params = {"name": name, "url": url, "suffix": suffix, "source": source}
    log.debug(f"Setting custom TMS with params: {tms_params}")
    self.sources["custom"] = tms_params
    self.source = "custom"

getFormat

getFormat()

Get the image format of the map tiles.

Returns:

Type Description
str

the upstream source for map tiles.

Source code in osm_fieldwork/basemapper.py
312
313
314
315
316
317
318
def getFormat(self):
    """Get the image format of the map tiles.

    Returns:
        (str): the upstream source for map tiles.
    """
    return self.sources[self.source]["suffix"]

getTiles

getTiles(zoom=None)

Get a list of tiles for the specifed zoom level.

Parameters:

Name Type Description Default
zoom int

The Zoom level of the desired map tiles

None

Returns:

Type Description
int

The total number of map tiles downloaded

Source code in osm_fieldwork/basemapper.py
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
def getTiles(
    self,
    zoom: int = None,
):
    """Get a list of tiles for the specifed zoom level.

    Args:
        zoom (int): The Zoom level of the desired map tiles

    Returns:
        (int): The total number of map tiles downloaded
    """
    info = get_cpu_info()
    cores = info["count"]

    self.tiles = list(mercantile.tiles(self.bbox[0], self.bbox[1], self.bbox[2], self.bbox[3], zoom))
    total = len(self.tiles)
    log.info("%d tiles for zoom level %d" % (len(self.tiles), zoom))
    chunk = round(len(self.tiles) / cores)
    queue.Queue(maxsize=cores)
    log.info("%d threads, %d tiles" % (cores, total))

    mirrors = [self.sources[self.source]]
    # epdb.st()
    if len(self.tiles) < chunk or chunk == 0:
        dlthread(self.base, mirrors, self.tiles, self.xy)
    else:
        with concurrent.futures.ThreadPoolExecutor(max_workers=cores) as executor:
            # results = []
            block = 0
            while block <= len(self.tiles):
                executor.submit(dlthread, self.base, mirrors, self.tiles[block : block + chunk], self.xy)
                # result = executor.submit(dlthread, self.base, mirrors, self.tiles[block : block + chunk], self.xy)
                # results.append(result)
                log.debug("Dispatching Block %d:%d" % (block, block + chunk))
                block += chunk
            executor.shutdown()
        # log.info("Had %r errors downloading %d tiles for data for %r" % (self.errors, len(tiles), Path(self.base).name))
        # for result in results:
        #     print(result.result())
    return len(self.tiles)

tileExists

tileExists(tile)

See if a map tile already exists.

Parameters:

Name Type Description Default
tile MapTile

The map tile to check for the existence of

required

Returns:

Type Description
bool

Whether the tile exists in the map tile cache

Source code in osm_fieldwork/basemapper.py
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
def tileExists(
    self,
    tile: MapTile,
):
    """See if a map tile already exists.

    Args:
        tile (MapTile): The map tile to check for the existence of

    Returns:
        (bool): Whether the tile exists in the map tile cache
    """
    filespec = f"{self.base}{tile[2]}/{tile[1]}/{tile[0]}.{self.sources[{self.source}]['suffix']}"
    if Path(filespec).exists():
        log.debug("%s exists" % filespec)
        return True
    else:
        log.debug("%s doesn't exists" % filespec)
        return False

options: show_source: false heading_level: 3

Create a basemap with given parameters.

Parameters:

Name Type Description Default
boundary str | BytesIO

The boundary for the area you want.

None
tms str

Custom TMS URL.

None
xy bool

Swap the X & Y coordinates when using a custom TMS if True.

False
outfile str

Output file name for the basemap.

None
zooms str

The Zoom levels, specified as a range (e.g., "12-17") or comma-separated levels (e.g., "12,13,14").

'12-17'
outdir str

Output directory name for tile cache.

None
source str

Imagery source, one of ["esri", "bing", "topo", "google", "oam"] (default is "esri").

'esri'
append bool

Whether to append to an existing file

False

Returns:

Type Description
None

None

Source code in osm_fieldwork/basemapper.py
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
def create_basemap_file(
    boundary=None,
    tms=None,
    xy=False,
    outfile=None,
    zooms="12-17",
    outdir=None,
    source="esri",
    append: bool = False,
) -> None:
    """Create a basemap with given parameters.

    Args:
        boundary (str | BytesIO, optional): The boundary for the area you want.
        tms (str, optional): Custom TMS URL.
        xy (bool, optional): Swap the X & Y coordinates when using a
            custom TMS if True.
        outfile (str, optional): Output file name for the basemap.
        zooms (str, optional): The Zoom levels, specified as a range
            (e.g., "12-17") or comma-separated levels (e.g., "12,13,14").
        outdir (str, optional): Output directory name for tile cache.
        source (str, optional): Imagery source, one of
            ["esri", "bing", "topo", "google", "oam"] (default is "esri").
        append (bool, optional): Whether to append to an existing file

    Returns:
        None
    """
    log.debug(
        "Creating basemap with params: "
        f"boundary={boundary} | "
        f"outfile={outfile} | "
        f"zooms={zooms} | "
        f"outdir={outdir} | "
        f"source={source} | "
        f"xy={xy} | "
        f"tms={tms}"
    )

    # Validation
    if not boundary:
        err = "You need to specify a boundary! (in-memory object or bbox)"
        log.error(err)
        raise ValueError(err)

    # Get all the zoom levels we want
    zoom_levels = list()
    if zooms:
        if zooms.find("-") > 0:
            start = int(zooms.split("-")[0])
            end = int(zooms.split("-")[1]) + 1
            x = range(start, end)
            for i in x:
                zoom_levels.append(i)
        elif zooms.find(",") > 0:
            levels = zooms.split(",")
            for level in levels:
                zoom_levels.append(int(level))
        else:
            zoom_levels.append(int(zooms))

    if not outdir:
        base = Path.cwd().absolute()
    else:
        base = Path(outdir).absolute()

    source = "custom" if tms else source
    tiledir = base / f"{source}tiles"
    # Make tile download directory
    tiledir.mkdir(parents=True, exist_ok=True)
    # Convert to string for other methods
    tiledir = str(tiledir)

    if not source and not tms:
        err = "You need to specify a source!"
        log.error(err)
        raise ValueError(err)

    basemap = BaseMapper(boundary, tiledir, source, xy)

    if tms:
        # Add TMS URL to sources for download
        basemap.customTMS(tms)

    # Args parsed, main code:
    tiles = list()
    for level in zoom_levels:
        # Download the tile directory
        basemap.getTiles(level)
        tiles += basemap.tiles

    if not outfile:
        log.info(f"No outfile specified, tile download finished: {tiledir}")
        return

    suffix = Path(outfile).suffix.lower()
    log.debug(f"Basemap output format: {suffix}")

    if any(substring in suffix for substring in ["sqlite", "mbtiles"]):
        outf = DataFile(outfile, basemap.getFormat(), append)
        if suffix == ".mbtiles":
            outf.addBounds(basemap.bbox)
            outf.addZoomLevels(zoom_levels)
        # Create output database and specify image format, png, jpg, or tif
        outf.writeTiles(tiles, tiledir)

    elif suffix == ".pmtiles":
        tile_dir_to_pmtiles(outfile, tiledir, basemap.bbox, source, xy)

    else:
        msg = f"Format {suffix} not supported"
        log.error(msg)
        raise ValueError(msg) from None
    log.info(f"Wrote {outfile}")

options: show_source: false heading_level: 3

Write PMTiles archive from tiles in the specified directory.

Parameters:

Name Type Description Default
outfile str

The output PMTiles archive file path.

required
tile_dir str | Path

The directory containing the tile images.

required
bbox tuple

Bounding box in format (min_lon, min_lat, max_lon, max_lat).

required
attribution str

Attribution string to include in PMTile archive.

required
is_xy bool

If the X/Y are swapped in the xyz URL.

False

Returns:

Type Description

None

Source code in osm_fieldwork/basemapper.py
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
def tile_dir_to_pmtiles(
    outfile: str,
    tile_dir: str | Path,
    bbox: tuple,
    attribution: str,
    is_xy=False,
):
    """Write PMTiles archive from tiles in the specified directory.

    Args:
        outfile (str): The output PMTiles archive file path.
        tile_dir (str | Path): The directory containing the tile images.
        bbox (tuple): Bounding box in format (min_lon, min_lat, max_lon, max_lat).
        attribution (str): Attribution string to include in PMTile archive.
        is_xy (bool): If the X/Y are swapped in the xyz URL.

    Returns:
        None
    """
    tile_dir = Path(tile_dir)

    # Get tile image format from the first file encountered
    first_file = next((file for file in tile_dir.rglob("*.*") if file.is_file()), None)

    if not first_file:
        err = "No tile files found in the specified directory. Aborting PMTile creation."
        log.error(err)
        raise ValueError(err)

    # FIXME passing as PMTileType[tile_format] does not work
    # tile_format = first_file.suffix.upper()

    # Get zoom levels from dirs
    zoom_levels = sorted([int(x.stem) for x in tile_dir.glob("*") if x.is_dir()])

    with open(outfile, "wb") as pmtile_file:
        writer = PMTileWriter(pmtile_file)

        for tile_path in tile_dir.rglob("*"):
            if tile_path.is_file() and tile_path.suffix in [".jpg", ".jpeg"]:
                tile_id = tileid_from_xyz_dir_path(tile_path, is_xy)

                with open(tile_path, "rb") as tile:
                    writer.write_tile(tile_id, tile.read())

        min_lon, min_lat, max_lon, max_lat = bbox

        # Write PMTile metadata
        writer.finalize(
            header={
                # "tile_type": TileType[tile_format.lstrip(".")],
                "tile_type": PMTileType.PNG,
                "tile_compression": PMTileCompression.NONE,
                "min_zoom": zoom_levels[0],
                "max_zoom": zoom_levels[-1],
                "min_lon_e7": int(min_lon * 10000000),
                "min_lat_e7": int(min_lat * 10000000),
                "max_lon_e7": int(max_lon * 10000000),
                "max_lat_e7": int(max_lat * 10000000),
                "center_zoom": zoom_levels[0],
                "center_lon_e7": int(min_lon + ((max_lon - min_lon) / 2)),
                "center_lat_e7": int(min_lat + ((max_lat - min_lat) / 2)),
            },
            metadata={"attribution": f{attribution}"},
        )

options: show_source: false heading_level: 3

Helper function to get the tile id from a tile in xyz directory structure.

TMS typically has structure z/y/x.png If the --xy flag is used during creation, then z/x/y is used.

Parameters:

Name Type Description Default
filepath Union[Path, str]

The path to tile image within the xyz directory.

required
is_xy bool

If the X/Y are swapped in the xyz URL.

False

Returns:

Name Type Description
int int

The globally defined tile id from the xyz definition.

Source code in osm_fieldwork/basemapper.py
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
def tileid_from_xyz_dir_path(filepath: Union[Path, str], is_xy: bool = False) -> int:
    """Helper function to get the tile id from a tile in xyz directory structure.

    TMS typically has structure z/y/x.png
    If the --xy flag is used during creation, then z/x/y is used.

    Args:
        filepath (Union[Path, str]): The path to tile image within the xyz directory.
        is_xy (bool): If the X/Y are swapped in the xyz URL.

    Returns:
        int: The globally defined tile id from the xyz definition.
    """
    # Extract the final 3 parts from the TMS file path
    tile_image_path = Path(filepath).parts[-3:]

    try:
        final_tile = int(Path(tile_image_path[-1]).stem)
    except ValueError as e:
        msg = f"Invalid tile path (cannot parse as int): {str(tile_image_path)}"
        log.error(msg)
        raise ValueError(msg) from e

    if is_xy:
        y = final_tile
        z, x = map(int, tile_image_path[:-1])
    else:
        x = final_tile
        z, y = map(int, tile_image_path[:-1])

    return zxy_to_tileid(z, x, y)

options: show_source: false heading_level: 3


Last update: July 12, 2024