Added code for building data files
This commit is contained in:
Martin Asprusten
parent
32ed39c61a
commit
4c914996a2
6
.gitignore
vendored
6
.gitignore
vendored
@ -1,6 +1,12 @@
|
||||
.gitattributes
|
||||
.vscode/
|
||||
dist/
|
||||
native/*
|
||||
node_modules/
|
||||
!native/route_search.cpp
|
||||
*.sh
|
||||
data_generation/database/
|
||||
data_generation/data/
|
||||
**/.ipynb_checkpoints/
|
||||
**/.jupyter_ystore.db
|
||||
**/*.dat
|
||||
|
||||
12
data_generation/compose.yaml
Normal file
12
data_generation/compose.yaml
Normal file
@ -0,0 +1,12 @@
|
||||
services:
|
||||
database:
|
||||
image: postgis/postgis:17-3.5
|
||||
volumes:
|
||||
- ./database/:/var/lib/postgresql/data/
|
||||
environment:
|
||||
POSTGRES_PASSWORD: database
|
||||
POSTGRES_USER: database
|
||||
POSTGRES_DB: database
|
||||
PGDATA: /var/lib/postgresql/data
|
||||
ports:
|
||||
- 5432:5432
|
||||
529
data_generation/generate_data_file.ipynb
Normal file
529
data_generation/generate_data_file.ipynb
Normal file
@ -0,0 +1,529 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e74d75eb-5f32-4587-a6ee-a928fe6571e9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Setting up the database\n",
|
||||
"Source data takes a lot of space, and might overwhelm the amount of RAM available on the computer. For this reason, we will use a database to work with the source data, so we can offload a bunch of the data from RAM to disk. The included compose.yaml file contains docker instructions to set up a PostgreSQL database with PostGIS. Run it first using\n",
|
||||
"```bash\n",
|
||||
"sudo docker compose up -d\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"The next cell creates the tables used in the database. This requires the psycopg2 library."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "269a55a2-6c49-4871-95dd-2317d8066a83",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import psycopg2\n",
|
||||
"\n",
|
||||
"def initialize_database():\n",
|
||||
"\tconnection = psycopg2.connect(database='database', host='localhost', user='database', password='database', port=5432)\n",
|
||||
"\t\n",
|
||||
"\twith connection:\n",
|
||||
"\t\twith connection.cursor() as cursor:\n",
|
||||
"\t\t\tcursor.execute('''\n",
|
||||
"\t\t\t\tCREATE TABLE IF NOT EXISTS nodes(\n",
|
||||
"\t\t\t\t\tid INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,\n",
|
||||
"\t\t\t\t\tposition GEOMETRY(PointZ) NOT NULL\n",
|
||||
"\t\t\t\t);\n",
|
||||
"\t\t\t''')\n",
|
||||
"\t\n",
|
||||
"\t\t\tcursor.execute('''\n",
|
||||
"\t\t\t\tCREATE TABLE IF NOT EXISTS edges(\n",
|
||||
"\t\t\t\t\tid INTEGER GENERATED ALWAYS AS IDENTITY,\n",
|
||||
"\t\t\t\t\tstart_point_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,\n",
|
||||
"\t\t\t\t\tend_point_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,\n",
|
||||
"\t\t\t\t\tmotorway BOOLEAN NOT NULL,\n",
|
||||
"\t\t\t\t\tspeed_limit INTEGER NOT NULL,\n",
|
||||
"\t\t\t\t\ttunnel BOOLEAN NOT NULL,\n",
|
||||
"\t\t\t\t\tpassable_same_direction BOOLEAN NOT NULL,\n",
|
||||
"\t\t\t\t\tpassable_opposite_direction BOOLEAN NOT NULL,\n",
|
||||
"\t\t\t\t\tCONSTRAINT no_loops CHECK (start_point_id IS DISTINCT FROM end_point_id)\n",
|
||||
"\t\t\t\t);\n",
|
||||
"\t\t\t\tCREATE UNIQUE INDEX IF NOT EXISTS no_repeats ON edges (LEAST(start_point_id, end_point_id), GREATEST(start_point_id, end_point_id));\n",
|
||||
"\t\t\t''')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3fd2a33f-cce8-485b-837b-e0d5a1edaa2c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Constants"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f846657a-a1e9-4c0a-8c82-22ca75ffd416",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Combine nodes closer than this distance from each other into the same node (in metres)\n",
|
||||
"MINIMUM_NODE_DISTANCE = 1\n",
|
||||
"# If we don't have a speed limit value in our data, use this as default\n",
|
||||
"DEFAULT_SPEED_LIMIT = 0\n",
|
||||
"# Support reading other data than just UTM33 for Norway\n",
|
||||
"SOURCE_DATA_EPSG_CODE = \"EPSG:32633\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "35f4509f-3d5a-43de-9120-132f3fc5c8e4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Reading source data\n",
|
||||
"Next, we need to read source data. The Norwegian Elveg 2.0 data is available in both SOSI and GML formats. I downloaded it in GML, since SOSI kind of seems old-fashioned. If you want to read other data types, write another conversion function below. Note that it is assumed that the units of the X, Y and Z coordinates are meters, so if necessary, convert the data into something like UML."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b9e34cb2-fdea-43f1-a09d-21f12b55ca3f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from dataclasses import dataclass\n",
|
||||
"from typing import Optional\n",
|
||||
"import xml.etree.ElementTree as ET\n",
|
||||
"import re\n",
|
||||
"\n",
|
||||
"@dataclass\n",
|
||||
"class Node:\n",
|
||||
"\tnode_id: str\n",
|
||||
"\t\n",
|
||||
"\tposition_x: float\n",
|
||||
"\tposition_y: float\n",
|
||||
"\tposition_z: Optional[float]\n",
|
||||
"\n",
|
||||
"@dataclass\n",
|
||||
"class Edge:\n",
|
||||
"\tstart_point_id: str\n",
|
||||
"\tend_point_id: str\n",
|
||||
"\n",
|
||||
"\tspeed_limit: int\n",
|
||||
"\tmotorway: bool\n",
|
||||
"\ttunnel: bool\n",
|
||||
"\tpassable_same_direction: bool\n",
|
||||
"\tpassable_opposite_direction: bool\n",
|
||||
"\n",
|
||||
"@dataclass\n",
|
||||
"class LoadedFile:\n",
|
||||
"\tnodes: list[Node]\n",
|
||||
"\tedges: list[Edge]\n",
|
||||
"\n",
|
||||
"\tx_extent: tuple[float, float]\n",
|
||||
"\ty_extent: tuple[float, float]\n",
|
||||
"\n",
|
||||
"def read_gml_file(file_location: str, print_progress: bool = False) -> LoadedFile:\n",
|
||||
"\tnodes: list[Node] = []\n",
|
||||
"\tedges: list[Edge] = []\n",
|
||||
"\n",
|
||||
"\tx_extent = (1e99, -1e99)\n",
|
||||
"\ty_extent = (1e99, -1e99)\n",
|
||||
"\t\n",
|
||||
"\tgeons = 'http://skjema.geonorge.no/SOSI/produktspesifikasjon/Elveg/2.0'\n",
|
||||
"\tgmlns = 'http://www.opengis.net/gml/3.2'\n",
|
||||
"\t\n",
|
||||
"\ttree = ET.parse(file_location)\n",
|
||||
"\troot = tree.getroot()\n",
|
||||
"\n",
|
||||
"\tmotorways: set[str] = set()\n",
|
||||
"\tspeed_limits: dict[str, int] = {}\n",
|
||||
"\n",
|
||||
"\tfor motorway in root.findall(f'.//{{{geons}}}Motorveg'):\n",
|
||||
"\t\tlocal_id = motorway.find(f'{{{geons}}}lineærPosisjon//{{{geons}}}lokalId').text\n",
|
||||
"\t\tmotorways.add(local_id)\n",
|
||||
"\n",
|
||||
"\tfor speed_limit in root.findall(f'.//{{{geons}}}Fartsgrense'):\n",
|
||||
"\t\tspeed_limit_value = int(speed_limit.find(f'.//{{{geons}}}fartsgrenseVerdi').text)\n",
|
||||
"\t\tlocal_id = speed_limit.find(f'{{{geons}}}lineærPosisjon//{{{geons}}}lokalId').text\n",
|
||||
"\t\tspeed_limits[local_id] = speed_limit_value\n",
|
||||
"\n",
|
||||
"\troad_links = root.findall(f'.//{{{geons}}}Veglenke')\n",
|
||||
"\ttotal_road_links = len(road_links)\n",
|
||||
"\tfor road_link_no, road_link in enumerate(road_links):\n",
|
||||
"\t\tif print_progress and (road_link_no % 10000 == 0 or road_link_no+1 == total_road_links):\n",
|
||||
"\t\t\tprint(f'\\rReading road link {road_link_no+1} of {total_road_links}'.ljust(50, ' '), end='')\n",
|
||||
"\t\t\n",
|
||||
"\t\troad_type = road_link.find(f'.//{{{geons}}}typeVeg').text\n",
|
||||
"\t\t# If the road type is stairs, car ferry or passenger ferry, skip it\n",
|
||||
"\t\tif road_type == 'trapp' or road_type == 'bilferje' or road_type == 'passasjerferje':\n",
|
||||
"\t\t\tcontinue\n",
|
||||
"\n",
|
||||
"\t\tlocal_id = road_link.find(f'.//{{{geons}}}lokalId').text\n",
|
||||
"\t\tmotorway = local_id in motorways\n",
|
||||
"\t\tspeed_limit = DEFAULT_SPEED_LIMIT if not local_id in speed_limits else speed_limits[local_id]\n",
|
||||
"\n",
|
||||
"\t\t# Check if our road is under the terrain, under the sea bottom, or under a glacier (i.e. a tunnel)\n",
|
||||
"\t\troad_link_medium = road_link.find(f'{{{geons}}}medium')\n",
|
||||
"\t\tif road_link_medium is not None:\n",
|
||||
"\t\t\troad_link_medium = road_link_medium.text\n",
|
||||
"\t\ttunnel = road_link_medium == 'underTerrenget' or road_link_medium == 'underSjøbunnen' or road_link_medium == 'underIsbre'\n",
|
||||
"\n",
|
||||
"\t\t# However, if the road is a walking or biking path, we won't categorise it as a tunnel (since what we really want to\n",
|
||||
"\t\t# avoid is road tunnels, and walking or biking path tunnels might just be underpasses or similar)\n",
|
||||
"\t\tif road_type == 'gangOgSykkelveg' or road_type == 'sykkelveg' or road_type == 'gangveg':\n",
|
||||
"\t\t\ttunnel = False\n",
|
||||
"\n",
|
||||
"\t\tpositions = road_link.find(f'.//{{{gmlns}}}posList')\n",
|
||||
"\t\tcoordinates = [float(x) for x in positions.text.split(' ')]\n",
|
||||
"\n",
|
||||
"\t\t# The lanes describe whether this is a one-way road or not\n",
|
||||
"\t\tlanes = road_link.find(f'.//{{{geons}}}feltoversikt')\n",
|
||||
"\t\tif lanes is None:\n",
|
||||
"\t\t\tpassable_same_direction = True\n",
|
||||
"\t\t\tpassable_opposite_direction = False\n",
|
||||
"\t\telse:\n",
|
||||
"\t\t\tlanes = lanes.text.split('#')\n",
|
||||
"\t\t\tpassable_same_direction = False\n",
|
||||
"\t\t\tpassable_opposite_direction = False\n",
|
||||
"\n",
|
||||
"\t\t\tfor lane in lanes:\n",
|
||||
"\t\t\t\tlane_number = int(re.search('^([0-9]+).*$', lane).group(1))\n",
|
||||
"\t\t\t\tif lane_number % 2 == 1:\n",
|
||||
"\t\t\t\t\tpassable_same_direction = True\n",
|
||||
"\t\t\t\telse:\n",
|
||||
"\t\t\t\t\tpassable_opposite_direction = True\n",
|
||||
"\n",
|
||||
"\t\t\treference_direction = road_link.find(f'.//{{{geons}}}referanseretning')\n",
|
||||
"\t\t\tif reference_direction is not None and reference_direction.text == 'mot':\n",
|
||||
"\t\t\t\tpassable_same_direction, passable_opposite_direction = passable_opposite_direction, passable_same_direction\n",
|
||||
"\n",
|
||||
"\t\tx_coordinates = coordinates[0::3]\n",
|
||||
"\t\ty_coordinates = coordinates[1::3]\n",
|
||||
"\t\tz_coordinates = coordinates[2::3]\n",
|
||||
"\n",
|
||||
"\t\tcurrent_node = None\n",
|
||||
"\n",
|
||||
"\t\tfor i in range(len(x_coordinates)):\n",
|
||||
"\t\t\tprevious_node = current_node\n",
|
||||
"\n",
|
||||
"\t\t\tcurrent_x = x_coordinates[i]\n",
|
||||
"\t\t\tcurrent_y = y_coordinates[i]\n",
|
||||
"\t\t\tcurrent_z = z_coordinates[i]\n",
|
||||
"\n",
|
||||
"\t\t\tx_extent = (min(current_x, x_extent[0]), max(current_x, x_extent[1]))\n",
|
||||
"\t\t\ty_extent = (min(current_y, y_extent[0]), max(current_y, y_extent[1]))\n",
|
||||
"\n",
|
||||
"\t\t\tnode_id = str(road_link_no).rjust(15, ' ') + str(i).rjust(15, ' ')\n",
|
||||
"\t\t\tcurrent_node = Node(node_id, current_x, current_y, current_z)\n",
|
||||
"\t\t\tnodes.append(current_node)\n",
|
||||
"\n",
|
||||
"\t\t\tif previous_node is not None:\n",
|
||||
"\t\t\t\tedge = Edge(previous_node.node_id, current_node.node_id, speed_limit, motorway, tunnel, passable_same_direction, passable_opposite_direction)\n",
|
||||
"\t\t\t\tedges.append(edge)\n",
|
||||
"\n",
|
||||
"\treturn LoadedFile(nodes, edges, x_extent, y_extent)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "963065c7-91ad-431c-9981-2c7efa65136f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Add loaded file to database\n",
|
||||
"Since the Norwegian GML data is something like 14 GB all together, we won't try to load it all into memory. Instead, we will write the files, one by one, into a database, and later we will read from the database to create a file. At least in the GML data, some nodes don't have good altitudes, so it is possible to provide a function for correcting strange altitudes. This will fire if an altitude is either None or less than -3000."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "43501a49-f2b6-441e-9a8c-8f6e16c77431",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import psycopg2\n",
|
||||
"import math\n",
|
||||
"from collections import defaultdict\n",
|
||||
"from typing import Optional, Callable\n",
|
||||
"\n",
|
||||
"def add_to_database(loadedFile: LoadedFile, correct_altitude_function: Optional[Callable[[float, float], float]] = None, print_progress: bool = False):\n",
|
||||
"\tconnection = psycopg2.connect(database='database', host='localhost', user='database', password='database', port=5432)\n",
|
||||
"\twith connection:\n",
|
||||
"\t\twith connection.cursor() as cursor:\n",
|
||||
"\t\t\t# Store previously defined nodes in a dict, using their geographical position as key\n",
|
||||
"\t\t\t# The keys are a tuple containing the point's X and Y coordinates, rounded to the closest MINIMUM_NODE_DISTANCE\n",
|
||||
"\t\t\t# The values are a lists containing up to several points, where each point is represented as a tuple containing\n",
|
||||
"\t\t\t# the point's database ID, and its actual X, Y and Z coordinates\n",
|
||||
"\t\t\tdefined_points: dict[tuple[int, int], list[tuple[int, float, float, float]]] = defaultdict(list)\n",
|
||||
"\t\t\n",
|
||||
"\t\t\t# Load all points that are already in the database that overlap with the extents of the current file, so we can combine previously added nodes\n",
|
||||
"\t\t\tmin_x = loadedFile.x_extent[0] - 2 * MINIMUM_NODE_DISTANCE\n",
|
||||
"\t\t\tmax_x = loadedFile.x_extent[1] + 2 * MINIMUM_NODE_DISTANCE\n",
|
||||
"\t\t\tmin_y = loadedFile.y_extent[0] - 2 * MINIMUM_NODE_DISTANCE\n",
|
||||
"\t\t\tmax_y = loadedFile.y_extent[1] + 2 * MINIMUM_NODE_DISTANCE\n",
|
||||
"\t\t\tpolygon_wkt = f'POLYGON(({min_x} {min_y}, {min_x} {max_y}, {max_x} {max_y}, {max_x} {min_y}, {min_x} {min_y}))'\n",
|
||||
"\t\t\tcursor.execute(f'SELECT id, ST_X(position), ST_Y(position), ST_Z(position) FROM nodes WHERE ST_Within(position, ST_GeomFromText(\\'{polygon_wkt}\\'))')\n",
|
||||
"\t\t\tfor node_id, x, y, z in cursor:\n",
|
||||
"\t\t\t\trounded_x = int(round(x / MINIMUM_NODE_DISTANCE))\n",
|
||||
"\t\t\t\trounded_y = int(round(y / MINIMUM_NODE_DISTANCE))\n",
|
||||
"\t\t\t\tdefined_points[(rounded_x, rounded_y)].append((node_id, x, y, z))\n",
|
||||
"\t\t\n",
|
||||
"\t\t\tnode_id_to_database_id: dict[str, int] = {}\n",
|
||||
"\t\t\n",
|
||||
"\t\t\t# Add all nodes to database\n",
|
||||
"\t\t\tnumber_of_nodes = len(loadedFile.nodes)\n",
|
||||
"\t\t\tfor node_number, node in enumerate(loadedFile.nodes):\n",
|
||||
"\t\t\t\tif print_progress and (node_number % 10000 == 0 or node_number+1 == number_of_nodes):\n",
|
||||
"\t\t\t\t\tprint(f'\\rStoring node {node_number+1} of {number_of_nodes}'.ljust(50, ' '), end='')\n",
|
||||
"\t\t\t\t# Correct node altitude if necessary and possible\n",
|
||||
"\t\t\t\tif correct_altitude_function is not None and (node.position_z is None or node.position_z < -3000):\n",
|
||||
"\t\t\t\t\tnode.position_z = correct_altitude_function(node.position_x, node.position_y)\n",
|
||||
"\n",
|
||||
"\t\t\t\t# If all else fails, just say the node is at sea level\n",
|
||||
"\t\t\t\tif node.position_z is None:\n",
|
||||
"\t\t\t\t\tnode.position_z = 0\n",
|
||||
"\t\t\t\t\n",
|
||||
"\t\t\t\trounded_x = int(round(node.position_x / MINIMUM_NODE_DISTANCE))\n",
|
||||
"\t\t\t\trounded_y = int(round(node.position_y / MINIMUM_NODE_DISTANCE))\n",
|
||||
"\t\t\n",
|
||||
"\t\t\t\t# Check if a node within minimum distance already exists\n",
|
||||
"\t\t\t\tclosest_node_id = None\n",
|
||||
"\t\t\t\tclosest_node_distance = 1e99\n",
|
||||
"\t\t\t\tfor dx in [-1, 0, 1]:\n",
|
||||
"\t\t\t\t\tfor dy in [-1, 0, 1]:\n",
|
||||
"\t\t\t\t\t\tfor other_node_id, other_node_x, other_node_y, other_node_z in defined_points[(rounded_x + dx, rounded_y + dy)]:\n",
|
||||
"\t\t\t\t\t\t\tdistance = math.sqrt((other_node_x - node.position_x)**2 + (other_node_y - node.position_y)**2 + (other_node_z - node.position_z)**2)\n",
|
||||
"\t\t\t\t\t\t\tif distance < closest_node_distance:\n",
|
||||
"\t\t\t\t\t\t\t\tclosest_node_distance = distance\n",
|
||||
"\t\t\t\t\t\t\t\tclosest_node_id = other_node_id\n",
|
||||
"\t\t\n",
|
||||
"\t\t\t\t# If a node within the closest distance already exists, store this\n",
|
||||
"\t\t\t\tif closest_node_distance < MINIMUM_NODE_DISTANCE:\n",
|
||||
"\t\t\t\t\tnode_id_to_database_id[node.node_id] = closest_node_id\n",
|
||||
"\t\t\t\telse:\n",
|
||||
"\t\t\t\t\tcursor.execute(f'INSERT INTO nodes(position) VALUES(ST_GeomFromText(\\'POINT({node.position_x} {node.position_y} {node.position_z})\\')) RETURNING ID;')\n",
|
||||
"\t\t\t\t\tnode_db_id = cursor.fetchone()[0]\n",
|
||||
"\t\t\t\t\tnode_id_to_database_id[node.node_id] = node_db_id\n",
|
||||
"\t\t\t\t\tdefined_points[(rounded_x, rounded_y)].append((node_db_id, node.position_x, node.position_y, node.position_z))\n",
|
||||
"\t\t\t\t\t\n",
|
||||
"\t\t\t# Next, add all the edges\n",
|
||||
"\t\t\t# Store which edges we have already added, to avoid violating constraint\n",
|
||||
"\t\t\texisting_edges = set()\n",
|
||||
"\t\t\tnumber_of_edges = len(loadedFile.edges)\n",
|
||||
"\t\t\tfor edge_number, edge in enumerate(loadedFile.edges):\n",
|
||||
"\t\t\t\tif print_progress and (edge_number % 10000 == 0 or edge_number+1 == number_of_edges):\n",
|
||||
"\t\t\t\t\tprint(f'\\rStoring edge {edge_number+1} of {number_of_edges}'.ljust(50, ' '), end='')\n",
|
||||
"\t\t\t\tstart_node_id = node_id_to_database_id[edge.start_point_id]\n",
|
||||
"\t\t\t\tend_node_id = node_id_to_database_id[edge.end_point_id]\n",
|
||||
"\t\t\t\t\n",
|
||||
"\t\t\t\tif start_node_id == end_node_id:\n",
|
||||
"\t\t\t\t\tcontinue\n",
|
||||
"\n",
|
||||
"\t\t\t\tadded_edge = (min(start_node_id, end_node_id), max(start_node_id, end_node_id))\n",
|
||||
"\t\t\t\tif added_edge in existing_edges:\n",
|
||||
"\t\t\t\t\tcontinue\n",
|
||||
"\t\t\t\texisting_edges.add(added_edge)\n",
|
||||
"\n",
|
||||
"\t\t\t\tcursor.execute(f'''\n",
|
||||
"\t\t\t\t\tINSERT INTO edges(\n",
|
||||
"\t\t\t\t\t\tstart_point_id,\n",
|
||||
"\t\t\t\t\t\tend_point_id,\n",
|
||||
"\t\t\t\t\t\tmotorway,\n",
|
||||
"\t\t\t\t\t\tspeed_limit,\n",
|
||||
"\t\t\t\t\t\ttunnel,\n",
|
||||
"\t\t\t\t\t\tpassable_same_direction,\n",
|
||||
"\t\t\t\t\t\tpassable_opposite_direction\n",
|
||||
"\t\t\t\t\t\t)\n",
|
||||
"\t\t\t\t\tVALUES(\n",
|
||||
"\t\t\t\t\t\t{start_node_id},\n",
|
||||
"\t\t\t\t\t\t{end_node_id},\n",
|
||||
"\t\t\t\t\t\t{edge.motorway},\n",
|
||||
"\t\t\t\t\t\t{edge.speed_limit},\n",
|
||||
"\t\t\t\t\t\t{edge.tunnel},\n",
|
||||
"\t\t\t\t\t\t{edge.passable_same_direction},\n",
|
||||
"\t\t\t\t\t\t{edge.passable_opposite_direction}\n",
|
||||
"\t\t\t\t\t);\n",
|
||||
"\t\t\t\t''')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "19525a68-a5a6-44a9-b427-f1fd98872556",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Write data file from database\n",
|
||||
"Try to pack the file as tightly as possible. Since we represent the road network as a mathematical graph, we need to store the edges and the nodes of the graph. We first store the nodes, and then the edges.\n",
|
||||
"\n",
|
||||
"First, a 4-byte integer says how many nodes are stored in the file. Then, each node consists of a 4-byte float for its x position, a 4-byte float for its y position, and a 2-byte integer for its height. Heights are represented as integers by adding 3000, rounding to the closest 10 centimetres, and then multiplying by 10. In this way, heights between around -3000 and +3500 metres can be stored. This needs some modification if you want to do this in a country with higher mountains than Norway. No ID is stored for the nodes; its position in the list is used as ID.\n",
|
||||
"\n",
|
||||
"Edges are similarly stored, using first a 4-byte integer that says how many edges are contained. Each edge then contains a 4-byte integer containing the ID of the node it starts from, and a 4-byte integer containing the ID of the node it leads to. Finally, a single byte contains the rest of the information about the edge. The first 4 bits of this byte contains the edge's speed limit, divided by 10. The next bit is a flag saying whether the node is passable when going against its defined direction (i.e. from the end node to the start node), and the bit after that says whether the edge is passable when going in the same direction as the edge. This is meant to represent whether the edge is a one-way street. The next bit is a flag for whether the edge is a tunnel, and then final bit is a flag saying whether the edge is a motorway.\n",
|
||||
"\n",
|
||||
"Even though it would make more sense to write the EPSG code of the coordinates at the beginning of the file, we'll write them at the end of the file for backwards compatibility. If this string is not present, the web site will assume the data is in EPSG:32633."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e740021a-0cd2-49bb-9751-34f76b03fa3c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import struct\n",
|
||||
"import psycopg2\n",
|
||||
"\n",
|
||||
"def write_data_file(file_name: str, print_progress: bool = False):\n",
|
||||
"\twith open(file_name, 'wb') as roads_file:\n",
|
||||
"\t\tconnection = psycopg2.connect(database='database', host='localhost', user='database', password='database', port=5432)\n",
|
||||
"\t\twith connection:\n",
|
||||
"\t\t\twith connection.cursor() as cursor:\n",
|
||||
"\t\t\t\tcursor.execute('SELECT COUNT(*) FROM nodes;')\n",
|
||||
"\t\t\t\t\n",
|
||||
"\t\t\t\t(number_of_points,) = cursor.fetchone()\n",
|
||||
"\t\t\t\troads_file.write(number_of_points.to_bytes(4, 'big'))\n",
|
||||
"\n",
|
||||
"\t\t\t\tcursor.execute('SELECT id, ST_X(position), ST_Y(position), ST_Z(position) FROM nodes ORDER BY id ASC')\n",
|
||||
"\t\t\t\tid_to_position_map = {}\n",
|
||||
"\t\t\t\tcounter = 0\n",
|
||||
"\t\t\t\tfor node_id, x, y, z in cursor:\n",
|
||||
"\t\t\t\t\tid_to_position_map[node_id] = counter\n",
|
||||
"\t\t\t\t\tcounter += 1\n",
|
||||
"\n",
|
||||
"\t\t\t\t\tif print_progress and (counter % 10000 == 0 or counter == number_of_points):\n",
|
||||
"\t\t\t\t\t\tprint(f'\\rWriting node {counter} of {number_of_points} to file'.ljust(50, ' '), end='')\n",
|
||||
"\n",
|
||||
"\t\t\t\t\troads_file.write(struct.pack('>f', x))\n",
|
||||
"\t\t\t\t\troads_file.write(struct.pack('>f', y))\n",
|
||||
"\n",
|
||||
"\t\t\t\t\tif z > 3500 or z < -3000:\n",
|
||||
"\t\t\t\t\t\tz = 0\n",
|
||||
"\n",
|
||||
"\t\t\t\t\theight_int = int(round(z + 3000) * 10)\n",
|
||||
"\t\t\t\t\troads_file.write(height_int.to_bytes(2, 'big'))\n",
|
||||
"\n",
|
||||
"\t\t\t\tcursor.execute('SELECT COUNT(*) FROM edges;')\n",
|
||||
"\t\t\t\t(number_of_edges,) = cursor.fetchone()\n",
|
||||
"\n",
|
||||
"\t\t\t\troads_file.write(number_of_edges.to_bytes(4, 'big'))\n",
|
||||
"\t\t\t\tcursor.execute('''\n",
|
||||
"\t\t\t\t\tSELECT\n",
|
||||
"\t\t\t\t\t\tstart_point_id,\n",
|
||||
"\t\t\t\t\t\tend_point_id,\n",
|
||||
"\t\t\t\t\t\tmotorway,\n",
|
||||
"\t\t\t\t\t\tspeed_limit,\n",
|
||||
"\t\t\t\t\t\ttunnel,\n",
|
||||
"\t\t\t\t\t\tpassable_same_direction,\n",
|
||||
"\t\t\t\t\t\tpassable_opposite_direction\n",
|
||||
"\t\t\t\t\tFROM edges ORDER BY id ASC;'''\n",
|
||||
"\t\t\t\t)\n",
|
||||
"\n",
|
||||
"\t\t\t\tcounter = 0\n",
|
||||
"\t\t\t\tfor start_point_id, end_point_id, motorway, speed_limit, tunnel, passable_same_direction, passable_opposite_direction in cursor:\n",
|
||||
"\t\t\t\t\troads_file.write(id_to_position_map[start_point_id].to_bytes(4, 'big'))\n",
|
||||
"\t\t\t\t\troads_file.write(id_to_position_map[end_point_id].to_bytes(4, 'big'))\n",
|
||||
"\n",
|
||||
"\t\t\t\t\tcounter += 1\n",
|
||||
"\t\t\t\t\tif print_progress and (counter % 10000 == 0 or counter == number_of_edges):\n",
|
||||
"\t\t\t\t\t\tprint(f'\\rWriting edge {counter} of {number_of_edges} to file'.ljust(50, ' '), end='')\n",
|
||||
"\n",
|
||||
"\t\t\t\t\tflags_integer = int(speed_limit / 10) << 4\n",
|
||||
"\t\t\t\t\tif motorway:\n",
|
||||
"\t\t\t\t\t\tflags_integer += 1\n",
|
||||
"\t\t\t\t\tif tunnel:\n",
|
||||
"\t\t\t\t\t\tflags_integer += 2\n",
|
||||
"\t\t\t\t\tif passable_same_direction:\n",
|
||||
"\t\t\t\t\t\tflags_integer += 4\n",
|
||||
"\t\t\t\t\tif passable_opposite_direction:\n",
|
||||
"\t\t\t\t\t\tflags_integer += 8\n",
|
||||
"\t\t\t\t\troads_file.write(flags_integer.to_bytes(1, 'big'))\n",
|
||||
"\n",
|
||||
"\t\t# Write EPSG code\n",
|
||||
"\t\tepsg_code_bytes = SOURCE_DATA_EPSG_CODE.encode('utf-8')\n",
|
||||
"\t\troads_file.write(len(epsg_code_bytes).to_bytes(4, 'big'))\n",
|
||||
"\t\troads_file.write(epsg_code_bytes)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6e718eae-a595-407f-8d0a-62d1b89206e7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Correct strange altitudes\n",
|
||||
"For the Norwegian GML data, we will query an elevation API for strange heights, and just assume that the road at that point is at terrain level."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d5b49d8e-1b17-4836-8ac7-2839a3ebb340",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"def get_actual_elevation(x, y):\n",
|
||||
"\tparameters = {\n",
|
||||
"\t\t'koordsys': '32633',\n",
|
||||
"\t\t'punkter': f'[[{x},{y}]]',\n",
|
||||
"\t\t'geojson': False\n",
|
||||
"\t}\n",
|
||||
"\treturn json.loads(requests.get('https://ws.geonorge.no/hoydedata/v1/punkt', params=parameters).text)[\"punkter\"][0][\"z\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "041cc46e-373b-44d2-afaa-fb4fe20575e4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Build data file from GML\n",
|
||||
"In this case, all the GML files are stored in the folder named data. Read all files in this folder, store them to database, and finally write the database to file."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c51308af-9ca1-4dc4-aed4-dc97100864ef",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"initialize_database()\n",
|
||||
"\n",
|
||||
"gml_files = os.listdir('./data')\n",
|
||||
"for file_number, gml_file in enumerate(gml_files):\n",
|
||||
"\tprint(f'\\rReading {gml_file}, number {file_number+1} of {len(gml_files)}'.ljust(50, ' '))\n",
|
||||
"\tdata_file = read_gml_file(f'./data/{gml_file}', print_progress=True)\n",
|
||||
"\tadd_to_database(data_file, get_actual_elevation, print_progress=True)\n",
|
||||
"write_data_file('roads.dat', print_progress=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1811fcd5-d2c8-4433-ad44-925dbbeca359",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.13.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -112,6 +112,8 @@ struct CurrentAreaSearch {
|
||||
};
|
||||
|
||||
// Data
|
||||
// For backwards compatibility, default code is EPSG:32633 (UTM zone 33)
|
||||
char const *epsgCode = "EPSG:32633";
|
||||
RoadNodeSet set;
|
||||
SearchResult lastSearchResult;
|
||||
std::set<uint32_t> excludedNodes;
|
||||
@ -214,10 +216,29 @@ void loadData(std::string filePath) {
|
||||
addLinkToRoadNode(set.roadNodes, fromPoint, toPoint, speedLimit, motorway, tunnel, passableSameDirection, passableOppositeDirection);
|
||||
addLinkToRoadNode(set.roadNodes, toPoint, fromPoint, speedLimit, motorway, tunnel, passableOppositeDirection, passableSameDirection);
|
||||
}
|
||||
|
||||
// Finally, check if the file contains a string describing the EPSG code of its coordinates. For backwards compatibility, this
|
||||
// field may not be there
|
||||
source.read(buffer, 4);
|
||||
if (!source.eof()) {
|
||||
uint32_t lengthOfString = ntohl(*reinterpret_cast<uint32_t*>(buffer));
|
||||
// I think it is technically a memory leak to create this buffer without eventually destroying it, but it should
|
||||
// only happen once, and it's tiny, so I don't care.
|
||||
char* stringBuffer = new char[lengthOfString+1];
|
||||
source.read(stringBuffer, lengthOfString);
|
||||
*(stringBuffer + lengthOfString) = 0;
|
||||
|
||||
epsgCode = stringBuffer;
|
||||
}
|
||||
|
||||
delete[] buffer;
|
||||
}
|
||||
|
||||
// Get the EPSG code
|
||||
std::string getEpsgCode() {
|
||||
return std::string(epsgCode);
|
||||
}
|
||||
|
||||
// Search functions
|
||||
JSNodeInfo findClosestNode(float positionX, float positionY) {
|
||||
float closestDistance = 1e99;
|
||||
@ -1002,6 +1023,7 @@ EMSCRIPTEN_BINDINGS(my_module) {
|
||||
emscripten::register_vector<AreaSearchEntry>("AreaSearchEntries");
|
||||
|
||||
emscripten::function("loadData", &loadData);
|
||||
emscripten::function("getEpsgCode", &getEpsgCode);
|
||||
emscripten::function("findClosestNode", &findClosestNode);
|
||||
emscripten::function("findAllPathsFromPoint", &findAllPathsFromPointJS);
|
||||
emscripten::function("getPath", &getPathJS);
|
||||
|
||||
@ -4,6 +4,7 @@ import proj4 from 'proj4';
|
||||
import type { Endpoint, Message, PathSegment, Polygon, SearchAreaResultEntry } from '../../interfaces';
|
||||
|
||||
var dataLoaded = false;
|
||||
var fileEpsg: string;
|
||||
var module: MainModule | undefined = undefined;
|
||||
|
||||
function sendErrorMessage(error: string): void {
|
||||
@ -19,9 +20,9 @@ function createCMultiPolygon(module: MainModule, polygons: Polygon[]): MultiPoly
|
||||
let cRing = new module.Ring();
|
||||
ring.coordinates.forEach(coordinate => {
|
||||
let polygonCoordinate = new module.PolygonCoordinate();
|
||||
let utmCoordinates = proj4('EPSG:4326', 'EPSG:32633', [coordinate.longitude, coordinate.latitude]);
|
||||
polygonCoordinate.x = utmCoordinates[0];
|
||||
polygonCoordinate.y = utmCoordinates[1];
|
||||
let fileCoordinates = proj4('EPSG:4326', fileEpsg, [coordinate.longitude, coordinate.latitude]);
|
||||
polygonCoordinate.x = fileCoordinates[0];
|
||||
polygonCoordinate.y = fileCoordinates[1];
|
||||
cRing.push_back(polygonCoordinate);
|
||||
});
|
||||
cPolygon.push_back(cRing);
|
||||
@ -36,8 +37,8 @@ function getAreaSearchResults(searchedNodes: AreaSearchEntries): SearchAreaResul
|
||||
for (var i = 0; i < searchedNodes.size(); i++) {
|
||||
let searchedNode = searchedNodes.get(i);
|
||||
if (searchedNode != null) {
|
||||
let utmCoordinates = [searchedNode.positionX, searchedNode.positionY];
|
||||
let lngLatCoordinates = proj4('EPSG:32633', 'EPSG:4326', utmCoordinates);
|
||||
let fileCoordinates = [searchedNode.positionX, searchedNode.positionY];
|
||||
let lngLatCoordinates = proj4(fileEpsg, 'EPSG:4326', fileCoordinates);
|
||||
searchResults.push({
|
||||
nodeId: searchedNode.nodeId,
|
||||
latitude: lngLatCoordinates[1],
|
||||
@ -61,6 +62,8 @@ onmessage = async (e) => {
|
||||
module.loadData('roads.dat');
|
||||
dataLoaded = true;
|
||||
|
||||
fileEpsg = module.getEpsgCode();
|
||||
|
||||
let returnMessage: Message = {dataLoaded: {}}
|
||||
postMessage(returnMessage);
|
||||
return;
|
||||
@ -74,11 +77,11 @@ onmessage = async (e) => {
|
||||
|
||||
if (message.findClosestNode != null) {
|
||||
let lngLatCoordinates = [message.findClosestNode.longitude, message.findClosestNode.latitude];
|
||||
let utmCoordinates = proj4('EPSG:4326', 'EPSG:32633', lngLatCoordinates);
|
||||
let node = module.findClosestNode(utmCoordinates[0], utmCoordinates[1]);
|
||||
let fileCoordinates = proj4('EPSG:4326', fileEpsg, lngLatCoordinates);
|
||||
let node = module.findClosestNode(fileCoordinates[0], fileCoordinates[1]);
|
||||
|
||||
let nodeUtmCoordinates = [node.positionX, node.positionY];
|
||||
let nodeLngLatCoordinates = proj4('EPSG:32633', 'EPSG:4326', nodeUtmCoordinates);
|
||||
let nodeFileCoordinates = [node.positionX, node.positionY];
|
||||
let nodeLngLatCoordinates = proj4(fileEpsg, 'EPSG:4326', nodeFileCoordinates);
|
||||
|
||||
let returnMessage: Message = {
|
||||
foundClosestNode: {
|
||||
@ -115,7 +118,7 @@ onmessage = async (e) => {
|
||||
return;
|
||||
}
|
||||
let coordinates = [nodeData.positionX, nodeData.positionY];
|
||||
let lngLatCoordinates = proj4('EPSG:32633', 'EPSG:4326', coordinates);
|
||||
let lngLatCoordinates = proj4(fileEpsg, 'EPSG:4326', coordinates);
|
||||
endpoints.push({
|
||||
nodeId: nodeData.nodeId,
|
||||
latitude: lngLatCoordinates[1],
|
||||
@ -146,11 +149,11 @@ onmessage = async (e) => {
|
||||
if (!previousPoint) {
|
||||
continue;
|
||||
}
|
||||
let previousUtm = [previousPoint.positionX, previousPoint.positionY];
|
||||
let currentUtm = [currentPoint.positionX, currentPoint.positionY];
|
||||
let previousFileCoordinates = [previousPoint.positionX, previousPoint.positionY];
|
||||
let currentFileCoordinates = [currentPoint.positionX, currentPoint.positionY];
|
||||
|
||||
let previousLngLat = proj4('EPSG:32633', 'EPSG:4326', previousUtm);
|
||||
let currentLngLat = proj4('EPSG:32633', 'EPSG:4326', currentUtm);
|
||||
let previousLngLat = proj4(fileEpsg, 'EPSG:4326', previousFileCoordinates);
|
||||
let currentLngLat = proj4(fileEpsg, 'EPSG:4326', currentFileCoordinates);
|
||||
|
||||
let speed = Math.max(previousPoint.currentSpeed, currentPoint.currentSpeed);
|
||||
let requiredSpeed = Math.max(previousPoint.requiredSpeed, currentPoint.requiredSpeed);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user