Split into Datastructure, Added ByteConverter, Full Split & Load
This commit is contained in:
232
OSMSplitter/ByteConverter.cs
Normal file
232
OSMSplitter/ByteConverter.cs
Normal file
@ -0,0 +1,232 @@
|
||||
using OSMDatastructure;
|
||||
|
||||
namespace OSMImporter;
|
||||
|
||||
public static class ByteConverter
|
||||
{
|
||||
#region Node
|
||||
/*
|
||||
* Node Byteform:
|
||||
* +-------------------------------+
|
||||
* | Latitude | Longitude | ConnectionSize| connection |
|
||||
* |---------------+---------------+---------------+---------------+
|
||||
* | 4 bytes | 4 bytes | 4 bytes | ConnectionSize|
|
||||
* | float | float | int | Connection |
|
||||
* +---------------+---------------+
|
||||
*/
|
||||
|
||||
public static byte[] GetBytes(Node node)
|
||||
{
|
||||
byte[] latitudeBytes = BitConverter.GetBytes(node.lat);
|
||||
byte[] longitudeBytes = BitConverter.GetBytes(node.lon);
|
||||
|
||||
Connection[] connections = node.GetConnections();
|
||||
Dictionary<byte[], byte[]> connectionsBytes = new ();
|
||||
int byteArraySize = latitudeBytes.Length + longitudeBytes.Length;
|
||||
|
||||
foreach (Connection connection in connections)
|
||||
{
|
||||
byte[] connectionBytes = ByteConverter.GetBytes(connection);
|
||||
byte[] connectionSizeBytes = BitConverter.GetBytes(connectionBytes.Length);
|
||||
connectionsBytes.Add(connectionSizeBytes, connectionBytes);
|
||||
byteArraySize += connectionSizeBytes.Length + connectionBytes.Length;
|
||||
}
|
||||
|
||||
byte[] retByteArray = new byte[byteArraySize];
|
||||
int offset = 0;
|
||||
|
||||
Array.Copy(latitudeBytes, 0, retByteArray, offset, latitudeBytes.Length);
|
||||
offset += latitudeBytes.Length;
|
||||
|
||||
Array.Copy(longitudeBytes, 0, retByteArray, offset, longitudeBytes.Length);
|
||||
offset += longitudeBytes.Length;
|
||||
|
||||
foreach (KeyValuePair<byte[], byte[]> connectionBytes in connectionsBytes)
|
||||
{
|
||||
Array.Copy(connectionBytes.Key, 0, retByteArray, offset, connectionBytes.Key.Length);
|
||||
offset += connectionBytes.Key.Length;
|
||||
|
||||
Array.Copy(connectionBytes.Value, 0, retByteArray, offset, connectionBytes.Value.Length);
|
||||
offset += connectionBytes.Value.Length;
|
||||
}
|
||||
|
||||
return retByteArray;
|
||||
}
|
||||
|
||||
public static Node ToNode(byte[] bytes)
|
||||
{
|
||||
int offset = 0;
|
||||
float lat = BitConverter.ToSingle(bytes, offset);
|
||||
offset += sizeof(float);
|
||||
|
||||
float lon = BitConverter.ToSingle(bytes, offset);
|
||||
offset += sizeof(float);
|
||||
|
||||
Node retNode = new Node(lat, lon);
|
||||
Console.WriteLine("Node Lat: {0} Lon: {1}", lat, lon);
|
||||
|
||||
while (offset < bytes.Length)
|
||||
{
|
||||
int size = BitConverter.ToInt32(bytes, offset);
|
||||
offset += sizeof(int);
|
||||
|
||||
byte[] connectionBytes = new byte[size];
|
||||
Array.Copy(bytes, offset, connectionBytes, 0, size);
|
||||
retNode.AddConnection(ByteConverter.ToConnection(connectionBytes));
|
||||
offset += connectionBytes.Length;
|
||||
}
|
||||
|
||||
return retNode;
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Connection
|
||||
/*
|
||||
* Connection Byteform:
|
||||
* +---------------+---------------+
|
||||
* | endId | endLatitude | endLongitude | tagBytes | tag | value |
|
||||
* |---------------+---------------+---------------+---------------+---------------+---------------+
|
||||
* | 8 bytes | 4 bytes | 4 bytes | 4 bytes | 1 byte | 1 byte |
|
||||
* | ulong | float | float | int | | |
|
||||
* +---------------+---------------+
|
||||
*/
|
||||
|
||||
public static byte[] GetBytes(Connection connection)
|
||||
{
|
||||
byte[] endIdBytes = BitConverter.GetBytes(connection.endNodeId);
|
||||
byte[] endLatBytes = BitConverter.GetBytes(connection.endNodeCoordinates.lat);
|
||||
byte[] endLonBytes = BitConverter.GetBytes(connection.endNodeCoordinates.lon);
|
||||
|
||||
Dictionary<Connection.tagType, object> connectionTags = connection.GetTags();
|
||||
byte[] tagBytes = BitConverter.GetBytes(connectionTags.Count * 2);
|
||||
|
||||
int byteArraySize = endIdBytes.Length + endLatBytes.Length + endLonBytes.Length + tagBytes.Length + connectionTags.Count * 2;
|
||||
byte[] retBytes = new byte[byteArraySize];
|
||||
int offset = 0;
|
||||
|
||||
Array.Copy(endIdBytes, 0, retBytes, offset, endIdBytes.Length);
|
||||
offset += endIdBytes.Length;
|
||||
|
||||
Array.Copy(endLatBytes, 0, retBytes, offset, endLatBytes.Length);
|
||||
offset += endLatBytes.Length;
|
||||
|
||||
Array.Copy(endLonBytes, 0 , retBytes, offset, endLonBytes.Length);
|
||||
offset += endLonBytes.Length;
|
||||
|
||||
Array.Copy(tagBytes, 0, retBytes, offset, tagBytes.Length);
|
||||
offset += tagBytes.Length;
|
||||
|
||||
foreach (KeyValuePair<Connection.tagType, object> tagKv in connectionTags)
|
||||
{
|
||||
retBytes[offset] = (byte)tagKv.Key;
|
||||
offset++;
|
||||
retBytes[offset] = (byte)tagKv.Value;
|
||||
offset++;
|
||||
}
|
||||
|
||||
return retBytes;
|
||||
}
|
||||
|
||||
public static Connection ToConnection(byte[] bytes)
|
||||
{
|
||||
int offset = 0;
|
||||
ulong endId = BitConverter.ToUInt64(bytes, offset);
|
||||
offset += sizeof(ulong);
|
||||
|
||||
float endLat = BitConverter.ToSingle(bytes, offset);
|
||||
offset += sizeof(float);
|
||||
|
||||
float endLon = BitConverter.ToSingle(bytes, offset);
|
||||
offset += sizeof(float);
|
||||
|
||||
Connection retConnection = new Connection(endId, new Coordinates(endLat, endLon));
|
||||
|
||||
int tagBytes = BitConverter.ToInt32(bytes, offset);
|
||||
Console.WriteLine("Connection EndId: {0} Lat: {1} Lon: {2} Tags: {3}", endId, endLat, endLon, tagBytes / 2);
|
||||
offset += sizeof(int);
|
||||
while (offset < bytes.Length)
|
||||
{
|
||||
byte key = bytes[offset];
|
||||
byte value = bytes[offset + 1];
|
||||
retConnection.AddTag(key, value);
|
||||
offset += 2;
|
||||
}
|
||||
|
||||
return retConnection;
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Region
|
||||
/*
|
||||
* Region Byteform:
|
||||
* +---------------+---------------+---------------+
|
||||
* | regionHash | nodeBytes | nodeId | node |
|
||||
* |---------------+---------------+---------------+---------------+
|
||||
* | 8 bytes | 4 bytes | 8 bytes | nodeBytes |
|
||||
* | ulong | int | ulong | Node |
|
||||
* +---------------+---------------+---------------+
|
||||
*/
|
||||
|
||||
public static byte[] GetBytes(Region region)
|
||||
{
|
||||
byte[] regionHashBytes = BitConverter.GetBytes(region.regionHash);
|
||||
int byteArraySize = regionHashBytes.Length;
|
||||
|
||||
Dictionary<byte[], byte[]> nodesBytes = new();
|
||||
foreach(KeyValuePair<ulong, Node> nodeKv in region.GetNodes())
|
||||
{
|
||||
byte[] nodeIdBytes = BitConverter.GetBytes(nodeKv.Key);
|
||||
byte[] nodeBytes = GetBytes(nodeKv.Value);
|
||||
nodesBytes.Add(nodeIdBytes, nodeBytes);
|
||||
byteArraySize += sizeof(int) + nodeIdBytes.Length + nodeBytes.Length;
|
||||
}
|
||||
|
||||
byte[] retBytes = new byte[byteArraySize];
|
||||
int offset = 0;
|
||||
|
||||
Array.Copy(regionHashBytes, 0, retBytes, offset, regionHashBytes.Length);
|
||||
offset += regionHashBytes.Length;
|
||||
|
||||
foreach (KeyValuePair<byte[], byte[]> nodeByteKv in nodesBytes)
|
||||
{
|
||||
Array.Copy(BitConverter.GetBytes(nodeByteKv.Value.Length), 0, retBytes, offset, sizeof(int));
|
||||
offset += sizeof(int);
|
||||
|
||||
Array.Copy(nodeByteKv.Key, 0, retBytes, offset, nodeByteKv.Key.Length);
|
||||
offset += nodeByteKv.Key.Length;
|
||||
|
||||
Array.Copy(nodeByteKv.Value, 0, retBytes, offset, nodeByteKv.Value.Length);
|
||||
offset += nodeByteKv.Value.Length;
|
||||
}
|
||||
|
||||
return retBytes;
|
||||
}
|
||||
|
||||
public static Region ToRegion(byte[] bytes)
|
||||
{
|
||||
int offset = 0;
|
||||
ulong regionHash = BitConverter.ToUInt64(bytes, offset);
|
||||
offset += sizeof(ulong);
|
||||
|
||||
Console.WriteLine("Region: {0}", regionHash);
|
||||
|
||||
Region retRegion = new Region(regionHash);
|
||||
|
||||
while (offset < bytes.Length)
|
||||
{
|
||||
int nodeBytesAmount = BitConverter.ToInt32(bytes, offset);
|
||||
offset += sizeof(int);
|
||||
|
||||
ulong nodeId = BitConverter.ToUInt64(bytes, offset);
|
||||
offset += sizeof(ulong);
|
||||
|
||||
byte[] nodeBytes = new byte[nodeBytesAmount];
|
||||
Array.Copy(bytes, offset, nodeBytes, 0, nodeBytesAmount);
|
||||
retRegion.AddNode(nodeId, ByteConverter.ToNode(nodeBytes));
|
||||
offset += nodeBytes.Length;
|
||||
}
|
||||
|
||||
return retRegion;
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
185
OSMSplitter/Importer.cs
Normal file
185
OSMSplitter/Importer.cs
Normal file
@ -0,0 +1,185 @@
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Xml;
|
||||
using OSMDatastructure;
|
||||
|
||||
namespace OSMImporter
|
||||
{
|
||||
public static class Importer
|
||||
{
|
||||
|
||||
public const float regionSize = 0.01f;
|
||||
private static readonly XmlReaderSettings readerSettings = new()
|
||||
{
|
||||
IgnoreWhitespace = true,
|
||||
IgnoreComments = true
|
||||
};
|
||||
|
||||
private static readonly NumberFormatInfo coordinateFormat = new NumberFormatInfo()
|
||||
{
|
||||
NumberDecimalSeparator = "."
|
||||
};
|
||||
|
||||
public static void Split(string xmlFilePath, string outputFolderPath)
|
||||
{
|
||||
if (!File.Exists(xmlFilePath))
|
||||
throw new FileNotFoundException();
|
||||
FileStream xmlFileStream = File.OpenRead(xmlFilePath);
|
||||
|
||||
Console.WriteLine("Reading ways once...");
|
||||
Dictionary<ulong, Node?> nodes = ReturnNodeIdDictionary(XmlReader.Create(xmlFileStream, readerSettings));
|
||||
|
||||
xmlFileStream.Position = 0;
|
||||
RegionManager regionManager = new RegionManager();
|
||||
|
||||
Console.WriteLine("Reading nodes...");
|
||||
LoadNodesIntoDictionary(ref nodes, XmlReader.Create(xmlFileStream, readerSettings), ref regionManager);
|
||||
|
||||
xmlFileStream.Position = 0;
|
||||
|
||||
Console.WriteLine("Reading ways twice...");
|
||||
CreateConnections(XmlReader.Create(xmlFileStream, readerSettings), ref nodes);
|
||||
|
||||
Console.WriteLine("Writing...");
|
||||
|
||||
foreach(Region region in regionManager.GetAllRegions())
|
||||
WriteRegion(region, outputFolderPath);
|
||||
}
|
||||
|
||||
private static Dictionary<ulong, Node?> ReturnNodeIdDictionary(XmlReader xmlReader)
|
||||
{
|
||||
Dictionary<ulong, Node?> retSet = new Dictionary<ulong, Node?>();
|
||||
while (xmlReader.ReadToFollowing("way"))
|
||||
{
|
||||
byte[] byteArray = Encoding.ASCII.GetBytes(xmlReader.ReadOuterXml());
|
||||
MemoryStream wayStream = new MemoryStream(byteArray);
|
||||
XmlReader wayReader = XmlReader.Create(wayStream);
|
||||
bool addNodes = false;
|
||||
while (wayReader.ReadToFollowing("tag") && !addNodes)
|
||||
{
|
||||
if (wayReader.GetAttribute("v")!.Equals("highway"))
|
||||
{
|
||||
addNodes = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (addNodes)
|
||||
{
|
||||
wayStream.Position = 0;
|
||||
wayReader = XmlReader.Create(wayStream);
|
||||
while (wayReader.ReadToFollowing("nd"))
|
||||
{
|
||||
retSet.TryAdd(Convert.ToUInt64(wayReader.GetAttribute("ref")!), null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
xmlReader.Close();
|
||||
return retSet;
|
||||
}
|
||||
|
||||
private static void LoadNodesIntoDictionary(ref Dictionary<ulong, Node?> nodes, XmlReader xmlReader, ref RegionManager regionManager)
|
||||
{
|
||||
while (xmlReader.ReadToFollowing("node"))
|
||||
{
|
||||
ulong id = Convert.ToUInt64(xmlReader.GetAttribute("id"));
|
||||
if (nodes.ContainsKey(id))
|
||||
{
|
||||
float lat = Convert.ToSingle(xmlReader.GetAttribute("lat")!, coordinateFormat);
|
||||
float lon = Convert.ToSingle(xmlReader.GetAttribute("lon")!, coordinateFormat);
|
||||
Node newNode = new Node(lat, lon);
|
||||
nodes[id] = newNode;
|
||||
regionManager.GetRegion(newNode).AddNode(id, newNode);
|
||||
}
|
||||
}
|
||||
xmlReader.Close();
|
||||
}
|
||||
|
||||
private static void CreateConnections(XmlReader xmlReader, ref Dictionary<ulong, Node?> nodes)
|
||||
{
|
||||
while (xmlReader.ReadToFollowing("way"))
|
||||
{
|
||||
byte[] byteArray = Encoding.ASCII.GetBytes(xmlReader.ReadOuterXml());
|
||||
MemoryStream wayStream = new MemoryStream(byteArray);
|
||||
XmlReader wayReader = XmlReader.Create(wayStream);
|
||||
Dictionary<string, string> tags = new Dictionary<string, string>();
|
||||
|
||||
bool addNodes = false;
|
||||
while (wayReader.ReadToFollowing("tag"))
|
||||
{
|
||||
if (wayReader.GetAttribute("v")!.Equals("highway"))
|
||||
{
|
||||
addNodes = true;
|
||||
}
|
||||
tags.Add(wayReader.GetAttribute("k")!, value: wayReader.GetAttribute("v")!);
|
||||
}
|
||||
|
||||
|
||||
if (addNodes)
|
||||
{
|
||||
HashSet<ulong> nodesInWay = new HashSet<ulong>();
|
||||
wayStream.Position = 0;
|
||||
wayReader = XmlReader.Create(wayStream);
|
||||
while (wayReader.ReadToFollowing("nd"))
|
||||
{
|
||||
nodesInWay.Add(Convert.ToUInt64(wayReader.GetAttribute("ref")));
|
||||
}
|
||||
ConnectNodesOfWay(nodes, nodesInWay.ToArray(), tags);
|
||||
}
|
||||
}
|
||||
xmlReader.Close();
|
||||
}
|
||||
|
||||
private static void ConnectNodesOfWay(Dictionary<ulong, Node?> nodes, ulong[] nodeIds,
|
||||
Dictionary<string, string> tags)
|
||||
{
|
||||
string oneWayString = tags.ContainsKey("oneway") ? tags["oneway"] : "no";
|
||||
bool oneWay = false;
|
||||
bool forward = true;
|
||||
switch (oneWayString)
|
||||
{
|
||||
case "yes":
|
||||
oneWay = true;
|
||||
break;
|
||||
case "-1":
|
||||
forward = false;
|
||||
break;
|
||||
}
|
||||
|
||||
for (int i = 0; i < nodeIds.Length - 1; i++)
|
||||
{
|
||||
if (oneWay)
|
||||
{
|
||||
if(nodes.ContainsKey(nodeIds[i]))
|
||||
nodes[nodeIds[i]]!.AddConnection(new Connection(nodeIds[i + 1], nodes[nodeIds[i + 1]]!, tags));
|
||||
if(nodes.ContainsKey(nodeIds[i + 1]))
|
||||
nodes[nodeIds[i + 1]]!.AddConnection(new Connection(nodeIds[i], nodes[nodeIds[i]]!, tags));
|
||||
}
|
||||
else if (forward)
|
||||
{
|
||||
if(nodes.ContainsKey(nodeIds[i]))
|
||||
nodes[nodeIds[i]]!.AddConnection(new Connection(nodeIds[i + 1], nodes[nodeIds[i + 1]]!, tags));
|
||||
}
|
||||
else
|
||||
{
|
||||
if(nodes.ContainsKey(nodeIds[i + 1]))
|
||||
nodes[nodeIds[i + 1]]!.AddConnection(new Connection(nodeIds[i], nodes[nodeIds[i]]!, tags));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void WriteRegion(Region region, string outputFolderPath)
|
||||
{
|
||||
if (!Directory.Exists(outputFolderPath))
|
||||
Directory.CreateDirectory(outputFolderPath);
|
||||
string fileName = region.regionHash.ToString();
|
||||
string fullPath = Path.Combine(outputFolderPath, fileName);
|
||||
if (!File.Exists(fullPath))
|
||||
File.Create(fullPath).Close();
|
||||
FileStream fileStream = new FileStream(fullPath, FileMode.Append);
|
||||
fileStream.Write(ByteConverter.GetBytes(region));
|
||||
fileStream.Close();
|
||||
}
|
||||
}
|
||||
}
|
||||
14
OSMSplitter/OSMSplitter.csproj
Normal file
14
OSMSplitter/OSMSplitter.csproj
Normal file
@ -0,0 +1,14 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<RootNamespace>OSMServer</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\OSMDatastructure\OSMDatastructure.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
31
OSMSplitter/RegionManager.cs
Normal file
31
OSMSplitter/RegionManager.cs
Normal file
@ -0,0 +1,31 @@
|
||||
using OSMDatastructure;
|
||||
|
||||
namespace OSMImporter
|
||||
{
|
||||
public class RegionManager
|
||||
{
|
||||
private readonly Dictionary<ulong, Region> _regions;
|
||||
|
||||
public RegionManager()
|
||||
{
|
||||
this._regions = new Dictionary<ulong, Region>();
|
||||
}
|
||||
|
||||
public Region GetRegion(Coordinates coordinates)
|
||||
{
|
||||
if(this._regions.ContainsKey(coordinates.GetRegionHash(Importer.regionSize)))
|
||||
return this._regions[coordinates.GetRegionHash(Importer.regionSize)];
|
||||
else
|
||||
{
|
||||
Region newRegion = new Region(coordinates, Importer.regionSize);
|
||||
this._regions.Add(newRegion.regionHash, value: newRegion);
|
||||
return newRegion;
|
||||
}
|
||||
}
|
||||
|
||||
public Region[] GetAllRegions()
|
||||
{
|
||||
return this._regions.Values.ToArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user