Mbtiles, python 3.0+, and docker support

.dockerignore

@@ -0,0 +1,2 @@
+/src/output/**/**
+/src/temp/**

.gitignore

@@ -1,2 +1,2 @@
-output/
+src/output/
-temp/
+src/temp/

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.8
+
+WORKDIR /app
+COPY src/requirements.txt ./
+
+RUN pip install -r requirements.txt
+
+# Bundle app source
+COPY src /app
+
+EXPOSE 8080
+CMD [ "python", "server.py" ]

README.md

@@ -16,11 +16,21 @@ This tiny python based script allows you to download map tiles from Google, Bing
 python server.py
 ```
 
-If your web browser doesn't open a map afterwards, navigate to `UI\index.htm` and open it manually. The output tiles will be in the `output\{timestamp}\` directory by default.
+Then open up your web browser and navigate to `http://localhost:8080`. The output map tiles will be in the `output\{timestamp}\` directory by default.
 
 ## Requirements
 
-Needs **Python 3.5** and a modern web browser. Other Python versions could work but aren't yet tested.
+Needs **Python 3.0+**, [Pillow](https://pypi.org/project/Pillow/) library, and a modern web browser. Other Python versions could work but aren't yet tested. If you can't install manually, try docker for easy setup.
+
+## Via Docker
+
+Docker is a pretty simple way to install and contain applications. [Install Docker on your system](https://www.docker.com/products/docker-desktop), and paste this on your command line:
+
+```sh
+docker run -v $PWD/output:/app/output/ -p 8080:8080 -it aliashraf/map-tiles-downloader
+```
+
+Now open the browser and head over to `http://localhost:8080`. The downloaded maps will be stored in the `output` directory.
 
 ## Purpose
 
@@ -31,6 +41,9 @@ I design map related things as a hobby, and often I have to work with offline ma
 - Super easy to use map UI to select region and options
 - Multi-threading to download tiles in parallel
 - Cross platform, use any OS as long as it has Python and a browser
+- Dockerfile available for easy setup
+- Supports 2x/Hi-Res/Retina/512x512 tiles my merging multiple tiles
+- Supports downloading to file as well as mbtile format
 - Select multiple zoom levels in one go
 - Ability to ignore tiles already downloaded
 - Specify any custom file name format
@@ -50,4 +63,4 @@ For latest releases and announcements, check out my site: [aliashraf.net](http:/
 This software is released under the [MIT License](LICENSE). Please read LICENSE for information on the
 software availability and distribution.
 
-Copyright (c) 2018 [Ali Ashraf](http://aliashraf.net)
+Copyright (c) 2020 [Ali Ashraf](http://aliashraf.net)

server.py

@@ -1,167 +0,0 @@
-#!/usr/bin/env python
-
-from http.server import BaseHTTPRequestHandler, HTTPServer
-from socketserver import ThreadingMixIn
-import threading
-
-from urllib.parse import urlparse
-from urllib.parse import parse_qs
-from urllib.parse import parse_qsl
-import urllib.request
-import cgi
-import uuid
-import random
-import string
-from cgi import parse_header, parse_multipart
-import argparse
-import uuid
-import random
-import time
-import json
-import shutil
-import ssl
-import glob
-import os
-import base64
-
-lock = threading.Lock()
-
-class serverHandler(BaseHTTPRequestHandler):
-		
-	def ensureDirectory(self, directory):
-
-		thisPath = os.path.join('output', directory)
-
-		lock.acquire()
-		try:
-
-			if not os.path.exists('temp'):
-				os.makedirs('temp')
-
-			if not os.path.exists('output'):
-				os.makedirs('output')
-
-
-			if not os.path.exists(thisPath):
-				os.makedirs(thisPath)
-
-		finally:
-			lock.release()
-
-		return thisPath
-
-	def randomString(self):
-		return uuid.uuid4().hex.upper()[0:6]
-
-	def downloadFile(self, url, destination):
-
-		code = 0
-
-		# monkey patching SSL certificate issue
-		# DONT use it in a prod/sensitive environment
-		ssl._create_default_https_context = ssl._create_unverified_context
-
-		try:
-			path, response = urllib.request.urlretrieve(url, destination)
-			code = 200
-		except urllib.error.URLError as e:
-			if not hasattr(e, "code"):
-				print(e)
-				code = -1
-			else:
-				code = e.code
-
-		return code
-		
-
-	def do_POST(self):
-
-		ctype, pdict = parse_header(self.headers['content-type'])
-		pdict['boundary'] = bytes(pdict['boundary'].encode('ascii'))
-		postvars = parse_multipart(self.rfile, pdict)
-
-		parts = urlparse(self.path)
-		if parts.path == '/download-tile':
-
-			x = int(postvars['x'][0])
-			y = int(postvars['y'][0])
-			z = int(postvars['z'][0])
-			quad = str(postvars['quad'][0].decode("utf-8"))
-			timestamp = int(postvars['timestamp'][0])
-			outputDirectory = str(postvars['outputDirectory'][0].decode("utf-8"))
-			outputFile = str(postvars['outputFile'][0].decode("utf-8"))
-			source = str(postvars['source'][0].decode("utf-8"))
-
-			replaceMap = {
-				"x": str(x),
-				"y": str(y),
-				"z": str(z),
-				"quad": quad,
-				"timestamp": str(timestamp),
-			}
-
-			for key, value in replaceMap.items():
-				newKey = str("{" + str(key) + "}")
-				outputDirectory = outputDirectory.replace(newKey, value)
-				outputFile = outputFile.replace(newKey, value)
-				source = source.replace(newKey, value)
-
-			result = {}
-
-			thisPath = self.ensureDirectory(outputDirectory);
-			filePath = os.path.join(thisPath, outputFile)
-
-			print("\n")
-
-			if os.path.isfile(filePath):
-				result["code"] = 200
-				result["message"] = 'Tile already exists'
-
-				print("EXISTS: " + filePath)
-
-			else:
-				tempFile = self.randomString() + ".png"
-				tempFilePath = os.path.join("temp", tempFile)
-
-				result["code"] = self.downloadFile(source, tempFilePath)
-
-				print("HIT: " + source + "\n" + "RETURN: " + str(result["code"]))
-
-				if os.path.isfile(tempFilePath):
-					shutil.copyfile(tempFilePath, filePath)
-					os.remove(tempFilePath)
-
-					result["message"] = 'Tile Downloaded'
-					print("SAVE: " + filePath)
-
-				else:
-					result["message"] = 'Download failed'
-
-
-			if os.path.isfile(filePath):
-				with open(filePath, "rb") as image_file:
-					result["image"] = base64.b64encode(image_file.read()).decode("utf-8")
-
-
-			self.send_response(200)
-			self.send_header("Access-Control-Allow-Origin", "*")
-			self.send_header("Content-Type", "application/json")
-			self.end_headers()
-			self.wfile.write(json.dumps(result).encode('utf-8'))
-			return
-		
-class serverThreadedHandler(ThreadingMixIn, HTTPServer):
-	"""Handle requests in a separate thread."""
-
-def run():
-	print('Starting Server...')
-	server_address = ('', 11291)
-	httpd = serverThreadedHandler(server_address, serverHandler)
-	print('Running Server...')
-
-	os.startfile('UI\\index.htm', 'open')
-	print("Open UI\\index.htm to view the application.")
-
-	httpd.serve_forever()
- 
-run()

src/UI/index.htm

@@ -5,6 +5,7 @@
 	<meta charset="utf-8"/>
     <meta name='viewport' content='initial-scale=1,maximum-scale=1,user-scalable=no' />
 
+	<!-- TODO replace jquery with react -->
 	<script src="jquery.min.js"></script> 
 
     <script src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.50.0/mapbox-gl.js'></script>
@@ -47,11 +48,11 @@
 			Search an Area
 		</div>
 
-		<form id='search-form'>
+		<form id='search-form' class='sidebar-section row'>
-			<p class="input-field col s12">
+			<div class="input-field col s12">
 				<input id="location-box" type="text" value="New York" />
 				<label for="location-box">Enter a location</label>
-			</p>
+			</div>
 		</form>
 
 
@@ -62,9 +63,11 @@
 			Select a Region
 		</div>
 
-		<p class='center-align'>
+		<div class='center-align sidebar-section'>
+			<p>
 				<a class="waves-effect waves-light z-depth-0 btn-small orange darken-3" id='rectangle-draw-button'>Draw a rectangle</a>
 			</p>
+		</div>
 
 		<div class='step-number'>
 			3
@@ -73,7 +76,7 @@
 			Configure
 		</div>
 
-		<p>
+		<div class='sidebar-section'>
 			<div class="row">
 				<div class="input-field col s6">
 					<input id="zoom-from-box" type="text" value='15'/>
@@ -103,7 +106,7 @@
 				<a class="waves-effect waves-light z-depth-0 btn-small orange darken-3" id='grid-preview-button'>Preview Grid</a>
 			</div>
 
-		</p>
+		</div>
 
 		<div class='step-number'>
 			4
@@ -112,21 +115,39 @@
 			<a href='javascript:void(0)' id='more-options-toggle'>More Options (+)</a>
 		</div>
 
-		<div style='display:none;' id='more-options'>
+		<div style='display:none;' id='more-options' class='sidebar-section row'>
-			<p class="input-field col s12">
+
+			<div class="input-field col s12">
+				<select id="output-scale" type="text">
+					<option value="1">1x</option>
+					<option value="2">2x</option>
+				</select>
+				<label for="output-scale">Output scale</label>
+			</div>
+
+			<div class="input-field col s12">
 				<input id="output-directory-box" type="text" value="{timestamp}">
 				<label for="output-directory-box">Output directory</label>
-			</p>
+			</div>
 
-			<p class="input-field col s12">
+			<div class="input-field col s12">
-				<input id="output-file-box" type="text" value="{x}-{y}-{z}.png">
+				<select id="output-type" type="text">
+					<option value="directory">Directory</option>
+					<option value="mbtiles">Mbtiles</option>
+					<option value="repo">Repo</option>
+				</select>
+				<label for="output-type">Output type</label>
+			</div>
+
+			<div class="input-field col s12">
+				<input id="output-file-box" type="text" value="{z}/{x}/{y}.png">
 				<label for="output-file-box">Output file</label>
-			</p>
+			</div>
 
-			<p class="input-field col s12">
+			<div class="input-field col s12">
 				<input id="parallel-threads-box" type="text" value="4">
 				<label for="parallel-threads-box">Parallel downloads</label>
-			</p>
+			</div>
 		</div>
 
 		<button class='waves-effect waves-light z-depth-0 btn-large cyan darken-2 bottom-button' id='download-button'>Download</button>
@@ -141,22 +162,25 @@
 		<div class="step-title">
 			Downloading tiles
 		</div>
-		<div class="hints">
+		<div class="hints sidebar-section">
 			Please wait...
 		</div>
 
-		<p id='progress-radial'>
+		<div class="sidebar-section">
+			<div id='progress-radial' class=''>
 	
-		</p>
+			</div>
 	
-		<p id='progress-subtitle'>
+			<p id='progress-subtitle' class=''>
 	
 			</p>
 	
 			<p class='tile-strip '>
 			</p>
 
-		<textarea id='log-view'></textarea>
+		</div>
+	
+		<textarea id='log-view' class="sidebar-section"></textarea>
 
 		<button class='waves-effect waves-light z-depth-0 btn-large red lighten-5 bottom-button' id='stop-button'>Stop</button>
 

src/UI/main.js

@@ -16,7 +16,7 @@ $(function() {
 		"Bing Maps Satellite": "http://ecn.t0.tiles.virtualearth.net/tiles/a{quad}.jpeg?g=129&mkt=en&stl=H",
 		"Bing Maps Hybrid": "http://ecn.t0.tiles.virtualearth.net/tiles/h{quad}.jpeg?g=129&mkt=en&stl=H",
 
-		"div-1": "",
+		"div-1B": "",
 
 		"Google Maps": "https://mt0.google.com/vt?lyrs=m&x={x}&s=&y={y}&z={z}",
 		"Google Maps Satellite": "https://mt0.google.com/vt?lyrs=s&x={x}&s=&y={y}&z={z}",
@@ -48,7 +48,7 @@ $(function() {
 
 		map = new mapboxgl.Map({
 			container: 'map-view',
-			style: 'mapbox://styles/mapbox/satellite-v9',
+			style: 'mapbox://styles/aliashraf/ck6lw9nr80lvo1ipj8zovttdx',
 			center: [-73.983652, 40.755024], 
 			zoom: 12
 		});
@@ -104,6 +104,18 @@ $(function() {
 			$("#more-options").toggle();
 		})
 
+		var outputFileBox = $("#output-file-box")
+		$("#output-type").change(function() {
+			var outputType = $("#output-type").val();
+			if(outputType == "mbtiles") {
+				outputFileBox.val("tiles.mbtiles")
+			} else if(outputType == "repo") {
+				outputFileBox.val("tiles.repo")
+			} else if(outputType == "directory") {
+				outputFileBox.val("{z}/{x}/{y}.png")
+			}
+		})
+
 	}
 
 	function initializeRectangleTool() {
@@ -232,7 +244,7 @@ $(function() {
 		return false;
 	}
 
-	function getGrid(zoomLevel) {
+	function getBounds() {
 
 		var coordinates = draw.getAll().features[0].geometry.coordinates[0];
 
@@ -240,23 +252,34 @@ $(function() {
 			return bounds.extend(coord);
 		}, new mapboxgl.LngLatBounds(coordinates[0], coordinates[0]));
 
+		return bounds;
+	}
+
+	function getGrid(zoomLevel) {
+
+		var bounds = getBounds();
+
 		var rects = [];
 
-		var TY    = lat2tile(bounds.getNorthEast().lat, zoomLevel);
+		var outputScale = $("#output-scale").val();
-		var LX   = long2tile(bounds.getSouthWest().lng, zoomLevel);
+		//var thisZoom = zoomLevel - (outputScale-1)
-		var BY = lat2tile(bounds.getSouthWest().lat, zoomLevel);
+		var thisZoom = zoomLevel
-		var RX  = long2tile(bounds.getNorthEast().lng, zoomLevel);
+
+		var TY    = lat2tile(bounds.getNorthEast().lat, thisZoom);
+		var LX   = long2tile(bounds.getSouthWest().lng, thisZoom);
+		var BY = lat2tile(bounds.getSouthWest().lat, thisZoom);
+		var RX  = long2tile(bounds.getNorthEast().lng, thisZoom);
 
 		for(var y = TY; y <= BY; y++) {
 			for(var x = LX; x <= RX; x++) {
 
-				var rect = getTileRect(x, y, zoomLevel);
+				var rect = getTileRect(x, y, thisZoom);
 
 				if(isTileInSelection(rect)) {
 					rects.push({
 						x: x,
 						y: y,
-						zoom: zoomLevel,
+						z: thisZoom,
 						rect: rect,
 					});
 				}
@@ -272,7 +295,7 @@ $(function() {
 
 		for(var z = getMinZoom(); z <= getMaxZoom(); z++) {
 			var grid = getGrid(z);
-			// TODO shuffle grid via a heuristic
+			// TODO shuffle grid via a heuristic (hamlet curve? :/)
 			allTiles = allTiles.concat(grid);
 		}
 
@@ -400,7 +423,7 @@ $(function() {
 		strip.prepend(image)
 	}
 
-	function startDownloading() {
+	async function startDownloading() {
 
 		if(draw.getAll().features.length == 0) {
 			M.toast({html: 'You need to select a region first.', displayLength: 3000})
@@ -426,8 +449,37 @@ $(function() {
 		var numThreads = parseInt($("#parallel-threads-box").val());
 		var outputDirectory = $("#output-directory-box").val();
 		var outputFile = $("#output-file-box").val();
+		var outputType = $("#output-type").val();
+		var outputScale = $("#output-scale").val();
 		var source = $("#source-box").val()
 
+		var bounds = getBounds();
+		var boundsArray = [bounds.getSouthWest().lng, bounds.getSouthWest().lat, bounds.getNorthEast().lng, bounds.getNorthEast().lat]
+		var centerArray = [bounds.getCenter().lng, bounds.getCenter().lat, getMaxZoom()]
+		
+		var data = new FormData();
+		data.append('minZoom', getMinZoom())
+		data.append('maxZoom', getMaxZoom())
+		data.append('outputDirectory', outputDirectory)
+		data.append('outputFile', outputFile)
+		data.append('outputType', outputType)
+		data.append('outputScale', outputScale)
+		data.append('source', source)
+		data.append('timestamp', timestamp)
+		data.append('bounds', boundsArray.join(","))
+		data.append('center', centerArray.join(","))
+
+		var request = await $.ajax({
+			url: "/start-download",
+			async: true,
+			timeout: 30 * 1000,
+			type: "post",
+			contentType: false,
+			processData: false,
+			data: data,
+			dataType: 'json',
+		})
+
 		let i = 0;
 		var iterator = async.eachLimit(allTiles, numThreads, function(item, done) {
 
@@ -437,17 +489,21 @@ $(function() {
 
 			var boxLayer = previewRect(item);
 
-			var url = "http://127.0.0.1:11291/download-tile";
+			var url = "/download-tile";
 
 			var data = new FormData();
 			data.append('x', item.x)
 			data.append('y', item.y)
-			data.append('z', item.zoom)
+			data.append('z', item.z)
-			data.append('quad', generateQuadKey(item.x, item.y, item.zoom))
+			data.append('quad', generateQuadKey(item.x, item.y, item.z))
 			data.append('outputDirectory', outputDirectory)
 			data.append('outputFile', outputFile)
+			data.append('outputType', outputType)
+			data.append('outputScale', outputScale)
 			data.append('timestamp', timestamp)
 			data.append('source', source)
+			data.append('bounds', boundsArray.join(","))
+			data.append('center', centerArray.join(","))
 
 			var request = $.ajax({
 				"url": url,
@@ -466,9 +522,9 @@ $(function() {
 
 				if(data.code == 200) {
 					showTinyTile(data.image)
-					logItem(item.x, item.y, item.zoom, data.message);
+					logItem(item.x, item.y, item.z, data.message);
 				} else {
-					logItem(item.x, item.y, item.zoom, data.code + " Error downloading tile");
+					logItem(item.x, item.y, item.z, data.code + " Error downloading tile");
 				}
 
 			}).fail(function(data, textStatus, errorThrown) {
@@ -477,7 +533,7 @@ $(function() {
 					return;
 				}
 
-				logItem(item.x, item.y, item.zoom, "Error while relaying tile");
+				logItem(item.x, item.y, item.z, "Error while relaying tile");
 				//allTiles.push(item);
 
 			}).always(function(data) {
@@ -495,7 +551,19 @@ $(function() {
 
 			requests.push(request);
 
-		}, function(err) {
+		}, async function(err) {
+
+			var request = await $.ajax({
+				url: "/end-download",
+				async: true,
+				timeout: 30 * 1000,
+				type: "post",
+				contentType: false,
+				processData: false,
+				data: data,
+				dataType: 'json',
+			})
+
 			updateProgress(allTiles.length, allTiles.length);
 			logItemRaw("All requests are done");
 
@@ -549,7 +617,6 @@ $(function() {
 
 	}
 
-
 	initializeMaterialize();
 	initializeSources();
 	initializeMap();

src/UI/style.css

@@ -7,31 +7,37 @@ html, body {
 }
 
 #max-height {
-	/*height: calc(100% - 120px);*/
 	height: 100%;
-	width: calc(100% - 300px);
+	width: calc(100% - 302px);
 	display: inline-block;
 	position: relative;
 }
 
 .sidebar {
-	width: 300px;
+	width: 340px;
 	height: calc(100% - 120px);
 	height: 100%;
 	position: absolute;
 	top: 0px;
 	right: 0px;
 	padding: 20px;
-	border-left: 5px solid #f1f2f6;
+	/* border-left: 5px solid #f1f2f6; */
+	overflow-x: visible;
+	overflow-y: auto;
+}
+
+.sidebar > .sidebar-section {
+	margin-left:40px;
+	margin-top: 0.5em;
+	margin-bottom: 0.5em;
+	width: calc(100% - 40px);
 }
 
 .bottom-button {
 	position: absolute;
-	bottom:15px;
+	bottom: 20px;
-	left: 0px;
+	left: 60px;
-	right: 0px;
+	width: 260px;
-	width: 90%;
-	margin: auto;
 	display: block;
 }
 
@@ -64,7 +70,7 @@ html, body {
 	display: inline-block;
 
 	padding-top: 2px;
-	margin-left: -40px;
+	/* margin-left: -40px; */
 	margin-right: 12px;
 	background: #00cec9;
 	width: 35px;
@@ -166,10 +172,8 @@ hr {
 }
 
 #log-view {
-	height: calc(100% - 450px);
+	height: calc(100% - 480px);
 	border: 1px solid #cfd8dc;
-
-
 	white-space: pre;
 	overflow-wrap: normal;
 	overflow-x: scroll;

src/file_writer.py

@@ -0,0 +1,74 @@
+import sqlite3
+import os
+import multiprocessing
+import io
+import json
+import shutil
+
+class FileWriter:
+
+	slicer = None
+	
+	def ensureDirectory(lock, directory):
+
+		lock.acquire()
+		try:
+
+			if not os.path.exists('temp'):
+				os.makedirs('temp')
+
+			if not os.path.exists('output'):
+				os.makedirs('output')
+
+			os.makedirs(directory, exist_ok=True)
+
+		finally:
+			lock.release()
+
+		return directory
+
+	@staticmethod
+	def addMetadata(lock, path, file, name, description, format, bounds, center, minZoom, maxZoom, profile="mercator", tileSize=256):
+
+		FileWriter.ensureDirectory(lock, path)
+
+		data = [
+			("name", name),
+			("description", description),
+			("format", format), 
+			("bounds", ','.join(map(str, bounds))), 
+			("center", ','.join(map(str, center))), 
+			("minzoom", minZoom), 
+			("maxzoom", maxZoom), 
+			("profile", profile), 
+			("tilesize", str(tileSize)), 
+			("scheme", "xyz"), 
+			("generator", "EliteMapper by Visor Dynamics"),
+			("type", "overlay"),
+			("attribution", "EliteMapper by Visor Dynamics"),
+		]
+		
+		with open(path + "/metadata.json", 'w+') as jsonFile:
+			json.dump(dict(data), jsonFile)
+
+		return
+
+	@staticmethod
+	def addTile(lock, filePath, sourcePath, x, y, z, outputScale):
+
+		fileDirectory = os.path.dirname(filePath)
+		FileWriter.ensureDirectory(lock, fileDirectory)
+		
+		shutil.copyfile(sourcePath, filePath)
+
+		return
+
+	@staticmethod
+	def exists(filePath, x, y, z):
+		return os.path.isfile(filePath)
+
+
+	@staticmethod
+	def close(lock, path, file, minZoom, maxZoom):
+		#TODO recalculate bounds and center
+		return

src/mbtiles_writer.py

@@ -0,0 +1,149 @@
+import sqlite3
+import os
+from os import listdir
+from os.path import isfile, join
+import multiprocessing
+from PIL import Image
+import io
+from utils import Utils
+
+class MbtilesWriter:
+	
+	def ensureDirectory(lock, directory):
+
+		lock.acquire()
+		try:
+
+			if not os.path.exists('temp'):
+				os.makedirs('temp')
+
+			if not os.path.exists('output'):
+				os.makedirs('output')
+
+			os.makedirs(directory, exist_ok=True)
+
+		finally:
+			lock.release()
+
+		return directory
+
+
+	@staticmethod
+	def addMetadata(lock, path, file, name, description, format, bounds, center, minZoom, maxZoom, profile="mercator", tileSize=256):
+
+		MbtilesWriter.ensureDirectory(lock, path)
+
+		connection = sqlite3.connect(file, check_same_thread=False)
+		c = connection.cursor()
+		c.execute("CREATE TABLE IF NOT EXISTS metadata (name text, value text);")
+		c.execute("CREATE TABLE IF NOT EXISTS tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);")
+		
+		try:
+			c.execute("CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row);")
+		except:
+			pass
+
+		try:
+			c.execute("CREATE UNIQUE INDEX metadata_name ON metadata (name);")
+		except:
+			pass
+		
+		connection.commit()
+		
+		try:
+			c.executemany("INSERT INTO metadata (name, value) VALUES (?, ?);", [
+				("name", name),
+				("description", description),
+				("format", format), 
+				("bounds", ','.join(map(str, bounds))), 
+				("center", ','.join(map(str, center))), 
+				("minzoom", minZoom), 
+				("maxzoom", maxZoom), 
+				("profile", profile), 
+				("tilesize", str(tileSize)), 
+				("scheme", "tms"), 
+				("generator", "Map Tiles Downloader via AliFlux"),
+				("type", "overlay"),
+				("attribution", "Map Tiles Downloader via AliFlux"),
+			])
+
+			connection.commit()
+		except:
+			pass
+
+		
+
+	@staticmethod
+	def addTile(lock, filePath, sourcePath, x, y, z, outputScale):
+
+		fileDirectory = os.path.dirname(filePath)
+		MbtilesWriter.ensureDirectory(lock, fileDirectory)
+
+		invertedY = (2 ** z) - y - 1
+
+		tileData = []
+		with open(sourcePath, "rb") as readFile:
+			tileData = readFile.read()
+		
+		lock.acquire()
+		try:
+
+			connection = sqlite3.connect(filePath, check_same_thread=False)
+			c = connection.cursor()
+			c.execute("INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?);", [
+				z, x, invertedY, tileData
+			])
+
+			connection.commit()
+
+		finally:
+			lock.release()
+
+
+		return
+
+	@staticmethod
+	def exists(filePath, x, y, z):
+		invertedY = (2 ** z) - y - 1
+
+		if(os.path.exists(filePath)):
+			
+			connection = sqlite3.connect(filePath, check_same_thread=False)
+			c = connection.cursor()
+
+			c.execute("SELECT COUNT(*) FROM tiles WHERE zoom_level = ? AND tile_column = ? AND tile_row = ? LIMIT 1", (z, x, invertedY))
+
+			result = c.fetchone()
+
+			if result[0] > 0:
+				return True
+
+		return False
+
+
+	@staticmethod
+	def close(lock, path, file, minZoom, maxZoom):
+
+		connection = sqlite3.connect(file, check_same_thread=False)
+		c = connection.cursor()
+
+		c.execute("SELECT min(tile_row), max(tile_row), min(tile_column), max(tile_column) from tiles WHERE zoom_level = ?", [maxZoom])
+
+		minY, maxY, minX, maxX = c.fetchone()
+		minY = (2 ** maxZoom) - minY - 1
+		maxY = (2 ** maxZoom) - maxY - 1
+
+		minLat, minLon = Utils.num2deg(minX, minY, maxZoom)
+		maxLat, maxLon = Utils.num2deg(maxX+1, maxY+1, maxZoom)
+
+		bounds = [minLon, minLat, maxLon, maxLat]
+		boundsString = ','.join(map(str, bounds))
+
+		center = [(minLon + maxLon)/2, (minLat + maxLat)/2, maxZoom]
+		centerString = ','.join(map(str, center))
+	
+		c.execute("UPDATE metadata SET value = ? WHERE name = 'bounds'", [boundsString])
+		c.execute("UPDATE metadata SET value = ? WHERE name = 'center'", [centerString])
+
+		connection.commit()
+

src/repo_writer.py

@@ -0,0 +1,87 @@
+import sqlite3
+import os
+from os import listdir
+from os.path import isfile, join
+import multiprocessing
+from PIL import Image
+import io
+from utils import Utils
+
+from mbtiles_writer import MbtilesWriter
+
+class RepoWriter(MbtilesWriter):
+
+	@staticmethod
+	def addMetadata(lock, path, file, name, description, format, bounds, center, minZoom, maxZoom, profile="mercator", tileSize=256):
+
+		RepoWriter.ensureDirectory(lock, path)
+
+		connection = sqlite3.connect(file, check_same_thread=False)
+		c = connection.cursor()
+		c.execute("CREATE TABLE IF NOT EXISTS metadata (name text, value text);")
+		c.execute("CREATE TABLE IF NOT EXISTS tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob, tile_cropped_data blob, pixel_left real, pixel_top real, pixel_right real, pixel_bottom real, has_alpha INTEGER);")
+		
+		try:
+			c.execute("CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row);")
+		except:
+			pass
+
+		try:
+			c.execute("CREATE UNIQUE INDEX metadata_name ON metadata (name);")
+		except:
+			pass
+		
+		connection.commit()
+		
+		c = connection.cursor()
+
+		
+		try:
+			c.executemany("INSERT INTO metadata (name, value) VALUES (?, ?);", [
+				("name", name),
+				("description", description),
+				("format", format), 
+				("bounds", ','.join(map(str, bounds))), 
+				("center", ','.join(map(str, center))), 
+				("minzoom", minZoom), 
+				("maxzoom", maxZoom), 
+				("profile", profile), 
+				("tilesize", str(tileSize)), 
+				("scheme", "tms"), 
+				("generator", "Map Tiles Downloader via AliFlux"),
+				("type", "overlay"),
+				("attribution", "Map Tiles Downloader via AliFlux"),
+			])
+
+			connection.commit()
+		except:
+			pass
+
+	@staticmethod
+	def addTile(lock, filePath, sourcePath, x, y, z, outputScale):
+
+		fileDirectory = os.path.dirname(filePath)
+		RepoWriter.ensureDirectory(lock, fileDirectory)
+
+		invertedY = (2 ** z) - y - 1
+
+		tileData = []
+		with open(sourcePath, "rb") as readFile:
+			tileData = readFile.read()
+		
+		lock.acquire()
+		try:
+
+			connection = sqlite3.connect(filePath, check_same_thread=False)
+			c = connection.cursor()
+			c.execute("INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data, tile_cropped_data, pixel_left, pixel_top, pixel_right, pixel_bottom, has_alpha) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", [
+				z, x, invertedY, None, tileData, 0, 0, 256 * outputScale, 256 * outputScale, 0
+			])
+
+			connection.commit()
+
+		finally:
+			lock.release()
+
+
+		return

src/requirements.txt

@@ -0,0 +1 @@
+Pillow==2.2.1

src/server.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from socketserver import ThreadingMixIn
+import threading
+
+from urllib.parse import urlparse
+from urllib.parse import parse_qs
+from urllib.parse import parse_qsl
+import urllib.request
+import cgi
+import uuid
+import random
+import string
+from cgi import parse_header, parse_multipart
+import argparse
+import uuid
+import random
+import time
+import json
+import shutil
+import ssl
+import glob
+import os
+import base64
+import mimetypes 
+
+from file_writer import FileWriter
+from mbtiles_writer import MbtilesWriter
+from repo_writer import RepoWriter
+from utils import Utils
+
+lock = threading.Lock()
+
+class serverHandler(BaseHTTPRequestHandler):
+		
+	def randomString(self):
+		return uuid.uuid4().hex.upper()[0:6]
+
+	def writerByType(self, type):
+		if(type == "mbtiles"):
+			return MbtilesWriter
+		elif(type == "repo"):
+			return RepoWriter
+		elif(type == "directory"):
+			return FileWriter
+
+	def do_POST(self):
+
+		ctype, pdict = cgi.parse_header(self.headers.get('Content-Type'))
+		#ctype, pdict = cgi.parse_header(self.headers['content-type'])
+		pdict['boundary'] = bytes(pdict['boundary'], "utf-8")
+				
+		content_len = int(self.headers.get('Content-length'))
+		pdict['CONTENT-LENGTH'] = content_len
+
+		postvars = cgi.parse_multipart(self.rfile, pdict)
+
+		parts = urlparse(self.path)
+		if parts.path == '/download-tile':
+
+			x = int(postvars['x'][0])
+			y = int(postvars['y'][0])
+			z = int(postvars['z'][0])
+			quad = str(postvars['quad'][0])
+			timestamp = int(postvars['timestamp'][0])
+			outputDirectory = str(postvars['outputDirectory'][0])
+			outputFile = str(postvars['outputFile'][0])
+			outputType = str(postvars['outputType'][0])
+			outputScale = int(postvars['outputScale'][0])
+			source = str(postvars['source'][0])
+
+			replaceMap = {
+				"x": str(x),
+				"y": str(y),
+				"z": str(z),
+				"quad": quad,
+				"timestamp": str(timestamp),
+			}
+
+			for key, value in replaceMap.items():
+				newKey = str("{" + str(key) + "}")
+				outputDirectory = outputDirectory.replace(newKey, value)
+				outputFile = outputFile.replace(newKey, value)
+
+			result = {}
+
+			filePath = os.path.join("output", outputDirectory, outputFile)
+
+			print("\n")
+
+			if self.writerByType(outputType).exists(filePath, x, y, z):
+				result["code"] = 200
+				result["message"] = 'Tile already exists'
+
+				print("EXISTS: " + filePath)
+
+			else:
+
+				tempFile = self.randomString() + ".png"
+				tempFilePath = os.path.join("temp", tempFile)
+
+				result["code"] = Utils.downloadFileScaled(source, tempFilePath, x, y, z, outputScale)
+
+				print("HIT: " + source + "\n" + "RETURN: " + str(result["code"]))
+
+				if os.path.isfile(tempFilePath):
+					self.writerByType(outputType).addTile(lock, filePath, tempFilePath, x, y, z, outputScale)
+
+					with open(tempFilePath, "rb") as image_file:
+						result["image"] = base64.b64encode(image_file.read()).decode("utf-8")
+
+					os.remove(tempFilePath)
+
+					result["message"] = 'Tile Downloaded'
+					print("SAVE: " + filePath)
+
+				else:
+					result["message"] = 'Download failed'
+
+
+			self.send_response(200)
+			# self.send_header("Access-Control-Allow-Origin", "*")
+			self.send_header("Content-Type", "application/json")
+			self.end_headers()
+			self.wfile.write(json.dumps(result).encode('utf-8'))
+			return
+			
+		elif parts.path == '/start-download':
+			outputType = str(postvars['outputType'][0])
+			outputScale = int(postvars['outputScale'][0])
+			outputDirectory = str(postvars['outputDirectory'][0])
+			outputFile = str(postvars['outputFile'][0])
+			minZoom = int(postvars['minZoom'][0])
+			maxZoom = int(postvars['maxZoom'][0])
+			timestamp = int(postvars['timestamp'][0])
+			bounds = str(postvars['bounds'][0])
+			boundsArray = map(float, bounds.split(","))
+			center = str(postvars['center'][0])
+			centerArray = map(float, center.split(","))
+
+			replaceMap = {
+				"timestamp": str(timestamp),
+			}
+
+			for key, value in replaceMap.items():
+				newKey = str("{" + str(key) + "}")
+				outputDirectory = outputDirectory.replace(newKey, value)
+				outputFile = outputFile.replace(newKey, value)
+
+			filePath = os.path.join("output", outputDirectory, outputFile)
+
+			self.writerByType(outputType).addMetadata(lock, os.path.join("output", outputDirectory), filePath, outputFile, "Map Tiles Downloader via AliFlux", "png", boundsArray, centerArray, minZoom, maxZoom, "mercator", 256 * outputScale)
+
+			result = {}
+			result["code"] = 200
+			result["message"] = 'Metadata written'
+
+			self.send_response(200)
+			# self.send_header("Access-Control-Allow-Origin", "*")
+			self.send_header("Content-Type", "application/json")
+			self.end_headers()
+			self.wfile.write(json.dumps(result).encode('utf-8'))
+			return
+			
+		elif parts.path == '/end-download':
+			outputType = str(postvars['outputType'][0])
+			outputScale = int(postvars['outputScale'][0])
+			outputDirectory = str(postvars['outputDirectory'][0])
+			outputFile = str(postvars['outputFile'][0])
+			minZoom = int(postvars['minZoom'][0])
+			maxZoom = int(postvars['maxZoom'][0])
+			timestamp = int(postvars['timestamp'][0])
+			bounds = str(postvars['bounds'][0])
+			boundsArray = map(float, bounds.split(","))
+			center = str(postvars['center'][0])
+			centerArray = map(float, center.split(","))
+
+			replaceMap = {
+				"timestamp": str(timestamp),
+			}
+
+			for key, value in replaceMap.items():
+				newKey = str("{" + str(key) + "}")
+				outputDirectory = outputDirectory.replace(newKey, value)
+				outputFile = outputFile.replace(newKey, value)
+
+			filePath = os.path.join("output", outputDirectory, outputFile)
+
+			self.writerByType(outputType).close(lock, os.path.join("output", outputDirectory), filePath, minZoom, maxZoom)
+
+			result = {}
+			result["code"] = 200
+			result["message"] = 'Downloaded ended'
+
+			self.send_response(200)
+			# self.send_header("Access-Control-Allow-Origin", "*")
+			self.send_header("Content-Type", "application/json")
+			self.end_headers()
+			self.wfile.write(json.dumps(result).encode('utf-8'))
+			return
+
+	def do_GET(self):
+
+
+		parts = urlparse(self.path)
+		path = parts.path.strip('/')
+		if path == "":
+			path = "index.htm"
+
+		file = os.path.join("./UI/", path)
+		mime = mimetypes.MimeTypes().guess_type(file)[0] 
+
+		self.send_response(200)
+		# self.send_header("Access-Control-Allow-Origin", "*")
+		self.send_header("Content-Type", mime)
+		self.end_headers()
+		
+		with open(file, "rb") as f:
+			self.wfile.write(f.read())
+		
+class serverThreadedHandler(ThreadingMixIn, HTTPServer):
+	"""Handle requests in a separate thread."""
+
+def run():
+	print('Starting Server...')
+	server_address = ('', 8080)
+	httpd = serverThreadedHandler(server_address, serverHandler)
+	print('Running Server...')
+
+	# os.startfile('UI\\index.htm', 'open')
+	print("Open http://localhost:8080/ to view the application.")
+
+	httpd.serve_forever()
+ 
+run()

src/utils.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+
+from urllib.parse import urlparse
+from urllib.parse import parse_qs
+from urllib.parse import parse_qsl
+import urllib.request
+import cgi
+import uuid
+import random
+import string
+from cgi import parse_header, parse_multipart
+import argparse
+import uuid
+import random
+import time
+import json
+import shutil
+import ssl
+import glob
+import os
+import base64
+import math
+
+from PIL import Image
+
+class Utils:
+	
+	@staticmethod
+	def randomString():
+		return uuid.uuid4().hex.upper()[0:6]
+
+	def getChildTiles(x, y, z):
+		childX = x * 2
+		childY = y * 2
+		childZ = z + 1
+
+		return [
+			(childX, childY, childZ),
+			(childX+1, childY, childZ),
+			(childX+1, childY+1, childZ),
+			(childX, childY+1, childZ),
+		]
+
+	def makeQuadKey(tile_x, tile_y, level):
+		quadkey = ""
+		for i in range(level):
+			bit = level - i
+			digit = ord('0')
+			mask = 1 << (bit - 1)  # if (bit - 1) > 0 else 1 >> (bit - 1)
+			if (tile_x & mask) is not 0:
+				digit += 1
+			if (tile_y & mask) is not 0:
+				digit += 2
+			quadkey += chr(digit)
+		return quadkey
+
+	@staticmethod
+	def num2deg(xtile, ytile, zoom):
+		n = 2.0 ** zoom
+		lon_deg = xtile / n * 360.0 - 180.0
+		lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * ytile / n)))
+		lat_deg = math.degrees(lat_rad)
+		return (lat_deg, lon_deg)
+
+	@staticmethod
+	def qualifyURL(url, x, y, z):
+
+		scale22 = 23 - (z * 2)
+
+		replaceMap = {
+			"x": str(x),
+			"y": str(y),
+			"z": str(z),
+			"scale:22": str(scale22),
+			"quad": Utils.makeQuadKey(x, y, z),
+		}
+
+		for key, value in replaceMap.items():
+			newKey = str("{" + str(key) + "}")
+			url = url.replace(newKey, value)
+
+		return url
+
+	@staticmethod
+	def mergeQuadTile(quadTiles):
+
+		width = 0
+		height = 0
+
+		for tile in quadTiles:
+			if(tile is not None):
+				width = quadTiles[0].size[0] * 2
+				height = quadTiles[1].size[1] * 2
+				break
+
+		if width == 0 or height == 0:
+			return None
+
+		canvas = Image.new('RGB', (width, height))
+
+		if quadTiles[0] is not None:
+			canvas.paste(quadTiles[0], box=(0,0))
+
+		if quadTiles[1] is not None:
+			canvas.paste(quadTiles[1], box=(width - quadTiles[1].size[0], 0))
+
+		if quadTiles[2] is not None:
+			canvas.paste(quadTiles[2], box=(width - quadTiles[2].size[0], height - quadTiles[2].size[1]))
+
+		if quadTiles[3] is not None:
+			canvas.paste(quadTiles[3], box=(0, height - quadTiles[3].size[1]))
+
+		return canvas
+
+	@staticmethod
+	def downloadFile(url, destination, x, y, z):
+
+		url = Utils.qualifyURL(url, x, y, z)
+
+		code = 0
+
+		# monkey patching SSL certificate issue
+		# DONT use it in a prod/sensitive environment
+		ssl._create_default_https_context = ssl._create_unverified_context
+
+		try:
+			path, response = urllib.request.urlretrieve(url, destination)
+			code = 200
+		except urllib.error.URLError as e:
+			if not hasattr(e, "code"):
+				print(e)
+				code = -1
+			else:
+				code = e.code
+
+		return code
+
+
+	@staticmethod
+	def downloadFileScaled(url, destination, x, y, z, outputScale):
+
+		if outputScale == 1:
+			return Utils.downloadFile(url, destination, x, y, z)
+
+		elif outputScale == 2:
+
+			childTiles = Utils.getChildTiles(x, y, z)
+			childImages = []
+
+			for childX, childY, childZ in childTiles:
+				
+				tempFile = Utils.randomString() + ".png"
+				tempFilePath = os.path.join("temp", tempFile)
+
+				code = Utils.downloadFile(url, tempFilePath, childX, childY, childZ)
+
+				if code == 200:
+					image = Image.open(tempFilePath)
+				else:
+					return code
+
+				childImages.append(image)
+			
+			canvas = Utils.mergeQuadTile(childImages)
+			canvas.save(destination, "PNG")
+			
+			return 200
+
+		#TODO implement custom scale
+
+			
+
+
+